diff --git a/.gitignore b/.gitignore index 90dac6cbc..75be106b5 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,10 @@ cython_debug/ # data folder data/ +# ...but the audio_analysis workflow ships checked-in package data (AudioSet +# category map, YAMNet class names) that must be tracked and wheel-included. +!src/senselab/audio/workflows/audio_analysis/data/ +!src/senselab/audio/workflows/audio_analysis/data/** # pdoc documentation docs/ diff --git a/README.md b/README.md index a983dfe62..ae5829843 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,57 @@ For more detailed information, check out our [**Documentation**](https://sensein - **senselab AI**: Interact with your data through an AI-based chatbot. The AI agent generates and runs senselab-based code for you, making exploration easier and giving you both the results and the code used to produce them (perfect for quick experiments or for users who prefer not to code). +--- + +## Adaptive audio analysis (uncertainty-driven) + +senselab can analyze a recording with its full task suite (diarization, scene/event tagging, quality +metrics, multi-model ASR + forced alignment, speaker embeddings), quantify **where** the models are +uncertain along three temporal axes — *presence* (is someone speaking?), *identity* (who?), and +*utterance* (what was said?) — and then **act on that uncertainty**: a deterministic, budgeted loop +re-processes only the uncertain regions (extra ASR models, embedding re-clustering, overlap +detection), fuses a consensus transcript/diarization, and explains any residual uncertainty instead +of hiding it. Design + results: `specs/20260723-225523-dynamic-uncertainty-workflow/`. + +It runs in two steps: + +```bash +# Step 1 — analyze: run every model on the recording (results are content-addressably +# cached, so re-runs are cheap). `--enhancement auto` adds a triage round 0 that skips +# the speech-enhancement pass on clean audio and stops early on silent recordings. +uv run python scripts/analyze_audio.py path/to/recording.wav --enhancement auto +# → artifacts/analyze_audio/_/ (per-task JSONs, 9 uncertainty +# parquets, Label Studio bundle, disagreements.json, timeline.png) + +# Step 2 — adapt: run the uncertainty-driven loop over that run directory. +uv run python scripts/adaptive_loop.py artifacts/analyze_audio/ \ + --cache-dir artifacts/analyze_audio_cache \ + --ground-truth path/to/labelstudio_export.json # optional: scores vs human labels +``` + +The loop writes, under the run directory (or `--out`): + +- `final/transcript.json` — consensus word-level transcript (family-weighted voting across all ASR + models) with speaker attribution, per-word confidence, and alternates where models disagree; +- `final/diarization.json` + `final/presence.parquet` — refined speaker segments (embedding + change-point + re-clustering repair) and the fused speech-presence track; +- `final/convergence.json` + `final/iterations.json` — what the loop did and why: every intervention + (fired / deferred / blocked) with trigger values and measured uncertainty deltas, budget + accounting, and regions marked `converged` / `irreducible` (with a machine-readable reason); +- `final/timeline.png` — ground truth (if given) vs presence / identity / utterance uncertainty per + round, interventions, and the confidence-colored fused words; +- `final/labelstudio_{tasks,config}.{json,xml}` + `disagreements_resolved.json` — the original Label + Studio bundle with `final__*` consensus tracks added, and the round-1 disagreements annotated with + their resolutions. + +Useful knobs (all thresholds/budgets/models live in a versioned policy file — +`src/senselab/audio/workflows/audio_analysis/adaptive/policy/default.yaml` — overridable via +`--policy my_policy.yaml`): `--max-rounds`, `--aggregator`, per-run intervention budgets, ASR +reserve/escalation pools, and identity-repair parameters. Runs are **deterministic**: identical +inputs + policy produce byte-identical decision logs. `HF_TOKEN` enables the gated +`pyannote/segmentation-3.0` overlap detector; without it the loop degrades gracefully and records +the skipped intervention in `convergence.json → next_actions`. + --- ## ⚠️ System Requirements diff --git a/pyproject.toml b/pyproject.toml index 586a8e468..b5531ce3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,4 +208,4 @@ skip = [ "*.cha", "*.ipynb" ] -ignore-words-list = ["senselab", "nd", "astroid", "wil", "SER", "te", "EXPRESSO", "VAI", "vie", "g2p"] +ignore-words-list = ["senselab", "nd", "astroid", "wil", "SER", "te", "EXPRESSO", "VAI", "vie", "g2p", "pres", "befores", "trough", "bellow", "coo", "patter", "AER"] diff --git a/scripts/adaptive_loop.py b/scripts/adaptive_loop.py new file mode 100644 index 000000000..ea9f787f1 --- /dev/null +++ b/scripts/adaptive_loop.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Adaptive uncertainty loop (prototype driver). + +Runs the loop from ``specs/20260723-225523-dynamic-uncertainty-workflow`` over a +completed ``analyze_audio.py`` run directory: round 1 ingests the run's +uncertainty parquets into the belief store (with a re-aggregation parity +check), rounds 2..K execute the policy-ranked intervention catalog (stream +election, hallucination adjudication, missed-speech correction, cache-replay +reserve-ASR escalation; live-backend rules defer with guard reasons), and the +final round fuses a consensus transcript / diarization / presence track with a +full decision audit trail. + +Usage: + uv run python scripts/adaptive_loop.py artifacts/e2e_runs/ \ + --cache-dir artifacts/analyze_audio_cache \ + --ground-truth ~/Downloads/updated-label-XXXX.json + +Works in minimal environments (no torch): heavy senselab imports are bypassed +via submodule loading when the full package cannot import. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_REPO_SRC = Path(__file__).resolve().parents[1] / "src" + + +def _ensure_light_importable() -> None: + """Make senselab importable when running from a checkout without installation. + + The package ``__init__`` chain is lazy (PEP 562 — see T046 in + ``specs/20260723-225523-dynamic-uncertainty-workflow/architecture-review.md``), + so the loop's pure submodules import without torch/transformers; all that is + needed here is the src/ path when senselab isn't pip-installed. + """ + try: + import senselab.audio.workflows.audio_analysis.aggregate # noqa: F401 + except ImportError: + if str(_REPO_SRC) not in sys.path: + sys.path.insert(0, str(_REPO_SRC)) + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("run_dir", type=Path, help="Completed analyze_audio run directory") + parser.add_argument("--cache-dir", type=Path, default=Path("artifacts/analyze_audio_cache")) + parser.add_argument("--policy", type=Path, default=None, help="Policy YAML overriding the default") + parser.add_argument("--out", type=Path, default=None, help="Output dir (default: run_dir)") + parser.add_argument("--max-rounds", type=int, default=3) + parser.add_argument("--aggregator", default=None, help="Override (default: from run's disagreements.json)") + parser.add_argument("--ground-truth", type=Path, default=None, help="Label Studio export JSON") + args = parser.parse_args(argv) + + if not (args.run_dir / "summary.json").exists(): + print(f"ERROR: {args.run_dir} is not an analyze_audio run dir (no summary.json)", file=sys.stderr) + return 2 + + _ensure_light_importable() + from senselab.audio.workflows.audio_analysis.adaptive.evaluate import evaluate_against_ground_truth + from senselab.audio.workflows.audio_analysis.adaptive.loop import run_adaptive_loop + + result = run_adaptive_loop( + args.run_dir, + cache_dir=args.cache_dir, + policy_path=args.policy, + out_dir=args.out, + max_rounds=args.max_rounds, + aggregator=args.aggregator, + ) + out_dir = Path(result["out_dir"]) + print(f"run_state: {result['run_state']} rounds: {result['rounds']}") + print("parity:", json.dumps(result["parity_check"])) + print( + f"interventions fired: {result['n_interventions_fired']} " + f"fused words: {result['n_words_fused']} stream: {result['fusion_stream']}" + ) + print(f"final/: {out_dir / 'final'}") + + # Visual timeline — best-effort sidecar (mirrors analyze_audio's timeline.png). + try: + from senselab.audio.workflows.audio_analysis.adaptive.plot import build_adaptive_timeline + + plot_path = build_adaptive_timeline(out_dir, gt_path=args.ground_truth, title=args.run_dir.name) + if plot_path is not None: + print(f"timeline: {plot_path}") + except Exception as exc: # noqa: BLE001 — plotting must never fail the run + print(f"warn: timeline plot failed: {exc!r}", file=sys.stderr) + + if args.ground_truth is not None: + eval_doc = evaluate_against_ground_truth( + out_dir=out_dir, gt_path=args.ground_truth, word_streams=result["word_streams"] + ) + t = eval_doc["transcript"] + print("\n── evaluation vs ground truth ──") + print(f"presence: {json.dumps(eval_doc['presence'])}") + print(f"fused WER: {t['fused']['wer']} (normalized: {t['fused']['wer_normalized']})") + for m, s in (t.get("per_model") or {}).items(): + print(f" {m}: WER {s['wer']} (normalized {s['wer_normalized']})") + print(f"diarization: {json.dumps(eval_doc['diarization'])}") + print(f"localization: {json.dumps(eval_doc['localization'])}") + print(f"eval.json: {out_dir / 'final' / 'eval.json'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/analyze_audio.py b/scripts/analyze_audio.py index 82751c4d8..626d512cf 100644 --- a/scripts/analyze_audio.py +++ b/scripts/analyze_audio.py @@ -26,22 +26,25 @@ **google/yamnet** (TF subprocess venv) speech_to_text (in defaults; mix of native-timestamp and post-aligned): - **openai/whisper-large-v3-turbo** (HFModel, 809M, multilingual; native timestamps) - **ibm-granite/granite-speech-3.3-8b** (~9B, EN + 7 translations; text-only, post-aligned by this script) + **nyralabs/CrisperWhisper2.0_turbo** (crisperwhisper CT2 subprocess venv; verbatim, native word + timestamps + per-word confidence) **nvidia/canary-qwen-2.5b** (NeMo SALM subprocess venv, 2.5B; text-only, post-aligned) **Qwen/Qwen3-ASR-1.7B** (qwen-asr subprocess venv, 1.7B; native word timestamps via Qwen3-ForcedAligner-0.6B companion) speech_to_text (additional, available via --asr-models): - openai/whisper-large-v3 (HFModel, 1.55B, multilingual; native timestamps) + openai/whisper-large-v3-turbo (HFModel, 809M, multilingual; native timestamps) + ibm-granite/granite-speech-3.3-8b (~9B, EN + 7 translations; text-only, post-aligned) openai/whisper-small (HFModel, 244M; native timestamps) nvidia/stt_en_conformer_ctc_large (NeMo subprocess venv, English-only; native CTC timestamps) Auto-align stage: every ASR model in --asr-models that returns text-only -ScriptLines (no native timestamps and no chunks) is automatically passed -through the multilingual MMS forced-aligner (--aligner-model, default -facebook/mms-1b-all, 1100+ languages via per-language adapters) to add -per-segment timestamps. Pass --no-align-asr to skip this and emit a +ScriptLines (no native timestamps and no chunks) is automatically force-aligned +to add per-word timestamps. The aligner is selectable via --aligner: 'qwen' +(default) uses Qwen3-ForcedAligner-0.6B in the qwen-asr subprocess venv; 'mms' +uses the multilingual facebook/mms-1b-all path (--aligner-model, 1100+ languages +via per-language adapters). ASR models with native timestamps (CrisperWhisper, +Qwen3-ASR) are never re-aligned. Pass --no-align-asr to skip this and emit a single full-audio TextArea region for those models in the LS export. The alignment cache is independent of the ASR cache (FR-024); changing the aligner re-runs only alignment, not ASR. @@ -136,6 +139,7 @@ from senselab.audio.tasks.speaker_diarization import diarize_audios from senselab.audio.tasks.speech_enhancement import enhance_audios from senselab.audio.tasks.speech_to_text import transcribe_audios +from senselab.audio.tasks.speech_to_text.qwen import QwenASR from senselab.audio.workflows.audio_analysis.harvesters import ( classification_window_top1 as _classification_window_top1, ) @@ -195,7 +199,43 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--no-enhancement", action="store_true", - help="Skip the enhanced-audio pass; only run on the resampled original.", + help="Skip the enhanced-audio pass; only run on the resampled original. Alias for --enhancement never.", + ) + parser.add_argument( + "--enhancement", + choices=("auto", "always", "never"), + default="always", + help=( + "Enhanced-pass policy (spec 20260723-225523 FR-003). 'always' preserves the historical " + "unconditional two-pass behavior; 'auto' runs a triage round 0 (segmentation-3.0 frame " + "posteriors + Brouhaha/DSP SNR) and runs the enhanced pass only when degraded speech is " + "found — and skips diarization/ASR/alignment/PPG entirely when no speech is found (FR-004); " + "'never' ≡ --no-enhancement." + ), + ) + parser.add_argument( + "--triage-speech-threshold", + type=float, + default=0.5, + help="Triage: per-window P(speech) at/above which a ~100 ms window counts as speech.", + ) + parser.add_argument( + "--triage-min-speech-s", + type=float, + default=0.3, + help="Triage: minimum total speech seconds for the run to proceed past round 0 (FR-004).", + ) + parser.add_argument( + "--triage-snr-floor-db", + type=float, + default=10.0, + help="Triage: SNR (dB) below which a speech window counts as degraded.", + ) + parser.add_argument( + "--triage-low-snr-fraction", + type=float, + default=0.4, + help="Triage: fraction of degraded speech windows at/above which the enhanced pass runs (FR-003).", ) parser.add_argument( "--diarization-models", @@ -215,24 +255,21 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--asr-models", nargs="+", default=[ - # Confirmed working through senselab today: - "openai/whisper-large-v3-turbo", - # Routes through the existing HF pipeline path with - # return_timestamps=False (timestamp-less HF model known-list); - # the script's auto-align stage adds per-segment timestamps via MMS: - "ibm-granite/granite-speech-3.3-8b", - # Both routed through dedicated subprocess venvs. Per-model - # failures are captured in JSON without aborting the run. + # CrisperWhisper 2.0 turbo — verbatim, word-level timestamps + native + # per-word confidence, via the crisperwhisper CT2 subprocess venv. + "nyralabs/CrisperWhisper2.0_turbo", + # Text-only NeMo SALM (subprocess venv); auto-aligned downstream. "nvidia/canary-qwen-2.5b", + # Native word timestamps via the bundled Qwen3-ForcedAligner companion + # (subprocess venv). Per-model failures are captured in JSON, non-fatal. "Qwen/Qwen3-ASR-1.7B", ], help=( - "ASR models. Defaults: Whisper Large v3 Turbo (native timestamps), " - "IBM Granite Speech 3.3 8B (text-only via HF pipeline; auto-aligned " - "by this script), NVIDIA Canary-Qwen 2.5B (text-only, auto-aligned), " - "and Qwen3-ASR 1.7B (native word-level timestamps via the bundled " - "forced-aligner companion). The script auto-aligns timestamp-less " - "ASR output via the multilingual aligner; pass --no-align-asr to skip." + "ASR models. Defaults: CrisperWhisper 2.0 turbo (verbatim, native word " + "timestamps + confidence), NVIDIA Canary-Qwen 2.5B (text-only, " + "auto-aligned), and Qwen3-ASR 1.7B (native word timestamps via the " + "bundled Qwen3-ForcedAligner companion). Timestamp-less ASR output is " + "auto-aligned by the script; pass --no-align-asr to skip." ), ) parser.add_argument( @@ -284,6 +321,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=0.48, help="YAMNet sliding-window hop, seconds (default: 0.48, matches YAMNet's native 50%% overlap hop).", ) + parser.add_argument( + "--scene-top-k", + type=int, + default=50, + help=( + "Number of AudioSet classes to persist per AST/YAMNet window (default: 50). " + "Feeds the presence-axis sound-source category masses (speech/people/machine/" + "environment); 50 captures essentially all of the softmax mass. Raise toward the " + "full label count (527 AST / 521 YAMNet) for the complete distribution at ~10x the " + "cache size; the top-1 label (speech-presence / YAMNet-veto) is unaffected either way." + ), + ) parser.add_argument( "--features-win-length", type=float, @@ -340,15 +389,30 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "ppgs subprocess venv (~1.4 GB)." ), ) + parser.add_argument( + "--aligner", + choices=("qwen", "mms"), + default="qwen", + help=( + "Forced-aligner backend for text-only ASR output (no native timestamps). " + "'qwen' (default) uses Qwen3-ForcedAligner-0.6B in the qwen-asr subprocess " + "venv; 'mms' uses the multilingual facebook/mms-1b-all path. ASR models " + "with native timestamps (CrisperWhisper, Qwen3-ASR) are never re-aligned." + ), + ) parser.add_argument( "--aligner-model", default="facebook/mms-1b-all", help=( - "Multilingual forced-alignment model used by the auto-align stage " - "(default: facebook/mms-1b-all, covers 1100+ languages via per-language " - "adapters). Override to swap MMS for another senselab-supported aligner." + "MMS forced-alignment model used when --aligner mms (default: " + "facebook/mms-1b-all, 1100+ languages via per-language adapters)." ), ) + parser.add_argument( + "--qwen-aligner-model", + default="Qwen/Qwen3-ForcedAligner-0.6B", + help="Qwen forced-aligner model id used when --aligner qwen (default: Qwen/Qwen3-ForcedAligner-0.6B).", + ) parser.add_argument( "--asr-language", default=None, @@ -378,20 +442,20 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--cross-stream-win-length", type=float, - default=0.5, + default=0.25, help=( - "Window length (seconds) for cross-stream / within-stream comparisons " - "(presence + identity axes). Default 0.5 s non-overlapping; finer grids " - "over-resolve the underlying signals (Whisper word-level ≈ 20 ms but " - "pyannote frames ≈ 62.5 ms, AST window 10.24 s). Utterance has its own grid — " - "see ``--utterance-win-length``." + "Window length (seconds) for the identity axis / cross-stream / within-stream " + "comparisons. Default 0.25 s overlapping — fine enough to localize sub-second " + "speaker turns (multi-speaker clips routinely have 0.3-1 s turns). Presence has " + "its own finer grid (``--presence-grid-*``) and utterance its own wider one " + "(``--utterance-win-length``)." ), ) parser.add_argument( "--cross-stream-hop-length", type=float, - default=0.5, - help="Hop between cross-stream comparison windows (default 0.5 s, non-overlapping; must be <= win-length).", + default=0.25, + help="Hop between identity/cross-stream windows (default 0.25 s; must be <= win-length).", ) parser.add_argument( "--utterance-win-length", @@ -414,21 +478,79 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "at least one bucket). Must be <= --utterance-win-length." ), ) + parser.add_argument( + "--no-scene-quality", + action="store_true", + help=( + "Disable the scene-quality signals (Brouhaha SNR/C50 + DSP clipping/bandwidth). " + "By default scene quality is REQUIRED: if Brouhaha cannot be loaded (e.g. gated " + "access not granted) the run fails loudly rather than silently emitting null " + "quality columns. Pass this flag to intentionally run without it." + ), + ) + parser.add_argument( + "--no-sound-sources", + action="store_true", + help="Disable the background sound-source category masses (speech/people/machine/environment).", + ) + parser.add_argument( + "--utterance-scene-coupling-weights", + type=float, + nargs=2, + metavar=("W_Q", "W_S"), + default=(0.5, 0.25), + help=( + "Scene-to-utterance coupling weights (FR-019): reported utterance uncertainty is " + "multiplied by 1 + W_Q * quality_snr + W_S * (src_machine + src_environment) over the " + "bucket's span, clipped to 1.0. The multiplier is recorded in the " + "scene_quality_coupling column and the pre-coupling value stays in " + "raw_aggregated_uncertainty. Defaults to 0.5 0.25; pass '0 0' to disable coupling." + ), + ) + parser.add_argument( + "--presence-grid-win-length", + type=float, + default=0.1, + help=( + "Window length (seconds) for the presence axis (default 0.1 s ≈ one phone). " + "Presence uses continuous frame posteriors, so a fine grid localizes brief " + "events (cough onset, inter-word breath) that a 0.5 s grid smears. Quality and " + "source columns are broadcast onto this grid." + ), + ) + parser.add_argument( + "--presence-grid-hop-length", + type=float, + default=0.02, + help="Hop between presence windows (default 0.02 s ≈ frame hop). Must be <= --presence-grid-win-length.", + ) parser.add_argument( "--embedding-window-s", type=float, - default=1.0, + default=0.5, help=( "Window length (seconds) for fixed-grid speaker-embedding extraction. " - "Defaults to 1.0 s — the smallest window that pairs reliably with the " - "0.5 s comparator bucket grid (one embedding per bucket center)." + "Defaults to 0.5 s: on conversational multi-speaker audio with short " + "turns, a 0.5 s window recovers the correct speaker count and roughly " + "doubles cluster-vs-truth agreement (ARI 0.70 vs 0.48 at 1.0 s on the " + "4-speaker validation clip) because 1.0 s windows straddle turn " + "boundaries and smear adjacent speakers together. The trade-off is that " + "turns shorter than the window (< 0.5 s) may not resolve as their own " + "cluster; raise toward 1.0 s for clean, long-form single/dual-speaker " + "audio where per-embedding robustness matters more than turn resolution." ), ) parser.add_argument( "--embedding-hop-s", type=float, - default=0.5, - help="Hop between embedding windows. Defaults to 0.5 s. Must be <= --embedding-window-s.", + default=0.25, + help=( + "Hop between embedding windows (default 0.25 s). A 0.25 s hop samples the " + "0.5 s window densely so speaker-change boundaries localize to ~0.25 s; on " + "the 4-speaker validation clip this flips the identity axis from inverted " + "(uncertainty dipping at speaker changes) to correct (peaking within ~15 ms " + "of the two clearest turn boundaries). Must be <= --embedding-window-s." + ), ) parser.add_argument( "--identity-same-speaker-floor", @@ -473,6 +595,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "to k-means automatically if a k fails." ), ) + parser.add_argument( + "--calibration-profile", + type=Path, + default=None, + help=( + "Scene-quality calibration profile JSON (US5, data-model §5): dB→[0,1] anchors for " + "SNR/C50 plus per-axis aggregator temperatures. Default: the bundled profile " + "(workflows/audio_analysis/data/scene_quality_calibration.json) when present, else the " + "documented uncalibrated defaults. Fit one with scripts/calibrate_scene_quality.py." + ), + ) parser.add_argument( "--uncertainty-aggregator", choices=UNCERTAINTY_AGGREGATORS, @@ -528,6 +661,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--utterance-win-length must be positive") if args.utterance_hop_length <= 0 or args.utterance_hop_length > args.utterance_win_length: parser.error("--utterance-hop-length must be positive and ≤ --utterance-win-length") + if args.presence_grid_win_length <= 0: + parser.error("--presence-grid-win-length must be positive") + if args.presence_grid_hop_length <= 0 or args.presence_grid_hop_length > args.presence_grid_win_length: + parser.error("--presence-grid-hop-length must be positive and ≤ --presence-grid-win-length") if not (0.0 <= args.phoneme_disagreement_threshold <= 1.0): parser.error("--phoneme-disagreement-threshold must be in [0, 1]") if args.diarization_boundary_shift_ms < 0: @@ -1452,6 +1589,291 @@ def _models_without_native_signal(summaries: dict[str, Any]) -> list[str]: return sorted(seen) +def _stage_diarization(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """Diarization models → ``summary["diarization"]["by_model"]`` (T012 stage).""" + summary, pass_dir, device = ctx["summary"], ctx["pass_dir"], ctx["device"] + summary["diarization"] = {"by_model": {}} + for model_id in args.diarization_models: + params = {"device": ctx["device_label"]} + outcome = run_task_cached( + f"diarization[{model_id}]", + diarize_audios, + [audio], + model=pick_dispatch_model(model_id, task="diarization"), + device=device, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("diarization", model_id, params), + provenance=ctx["prov"]("diarization", model_id, params), + ) + summary["diarization"]["by_model"][model_id] = outcome + write_json(pass_dir / "diarization" / f"{_safe(model_id)}.json", outcome) + + +def _stage_scene(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """AST + YAMNet classification (+ same-grid agreement sidecar) (T012 stage).""" + summary, pass_dir, device = ctx["summary"], ctx["pass_dir"], ctx["device"] + ast_win, ast_hop = args.ast_win_length, args.ast_hop_length + yam_win, yam_hop = args.yamnet_win_length, args.yamnet_hop_length + + if "ast" not in args.skip: + params = { + "win_length": ast_win, + "hop_length": ast_hop, + "top_k": args.scene_top_k, + "device": ctx["device_label"], + } + summary["ast"] = run_task_cached( + "ast", + classify_audios, + [audio], + model=HFModel(path_or_uri=args.ast_model), + device=device, + win_length=ast_win, + hop_length=ast_hop, + top_k=args.scene_top_k, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("ast", args.ast_model, params), + provenance=ctx["prov"]("ast", args.ast_model, params), + ) + summary["ast"]["window"] = {"win_length": ast_win, "hop_length": ast_hop} + write_json(pass_dir / "ast.json", summary["ast"]) + + if "yamnet" not in args.skip: + # YAMNet runs in senselab's TF subprocess venv (same pattern as NeMo + # Sortformer). senselab.classify_audios's `_is_yamnet()` dispatcher + # matches on the raw model-id *string*, not on a SenselabModel wrapper — + # passing HFModel here would fail validation (yamnet isn't on HF). + params = {"win_length": yam_win, "hop_length": yam_hop, "top_k": args.scene_top_k} + summary["yamnet"] = run_task_cached( + "yamnet", + classify_audios, + [audio], + model=args.yamnet_model, + win_length=yam_win, + hop_length=yam_hop, + top_k=args.scene_top_k, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("yamnet", args.yamnet_model, params), + provenance=ctx["prov"]("yamnet", args.yamnet_model, params), + ) + summary["yamnet"]["window"] = {"win_length": yam_win, "hop_length": yam_hop} + write_json(pass_dir / "yamnet.json", summary["yamnet"]) + + # If both ran on the same grid, emit a side-by-side comparison. + if ( + "ast" not in args.skip + and "yamnet" not in args.skip + and summary.get("ast", {}).get("status") == "ok" + and summary.get("yamnet", {}).get("status") == "ok" + and ast_win == yam_win + and ast_hop == yam_hop + ): + agreement = _scene_agreement( + ast_result=summary["ast"]["result"], + yamnet_result=summary["yamnet"]["result"], + win_length=ast_win, + hop_length=ast_hop, + ) + summary["scene_agreement"] = agreement + write_json(pass_dir / "scene_agreement.json", agreement) + + +def _stage_features(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """Temporal features (openSMILE / parselmouth / squim) + parquet sidecars (T012 stage).""" + summary, pass_dir, device = ctx["summary"], ctx["pass_dir"], ctx["device"] + feat_params: dict[str, Any] = { + "opensmile": "LowLevelDescriptors@native", + "parselmouth": "windowed", + "torchaudio_squim": "windowed", + "device": ctx["device_label"], + "win_length": args.features_win_length, + "hop_length": args.features_hop_length, + } + summary["features"] = run_task_cached( + "features", + extract_temporal_features, + audio, + win_length=args.features_win_length, + hop_length=args.features_hop_length, + device=device, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("features", None, feat_params), + provenance=ctx["prov"]("features", None, feat_params), + ) + # Each backend writes its own parquet sidecar — they have + # different columns and different time grids (opensmile LLD is + # native ~10 ms; parselmouth/torchaudio_squim follow the + # ``--features-*-length`` window). Cache outcome metadata stays + # in JSON for inspection parity with the other tasks. + result = summary["features"].get("result") or {} + if isinstance(result, dict): + try: + import pandas as pd + + feat_dir = pass_dir / "features" + feat_dir.mkdir(parents=True, exist_ok=True) + for backend, rows in result.items(): + if not rows: + continue + pd.DataFrame(rows).to_parquet(feat_dir / f"{backend}.parquet", index=False) + except Exception as exc: # noqa: BLE001 — best-effort sidecar + print(f" [features] warn: parquet write failed: {exc!r}", file=sys.stderr) + write_json(pass_dir / "features.json", {**summary["features"], "result": "see features/*.parquet"}) + + +def _stage_asr(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """ASR models → ``summary["asr"]["by_model"]`` (T012 stage).""" + summary, pass_dir, device = ctx["summary"], ctx["pass_dir"], ctx["device"] + summary["asr"] = {"by_model": {}} + for model_id in args.asr_models: + asr_params: dict[str, Any] = {"device": ctx["device_label"]} + extra_kwargs: dict[str, Any] = {} + # Qwen3-ASR ships its own forced-aligner companion model; allow + # opt-out so the script's MMS auto-align stage can take over. + if model_id.startswith("Qwen/Qwen3-ASR") and args.qwen_asr_no_timestamps: + extra_kwargs["return_timestamps"] = False + asr_params["return_timestamps"] = False + outcome = run_task_cached( + f"asr[{model_id}]", + transcribe_audios, + [audio], + model=pick_dispatch_model(model_id, task="asr"), + device=device, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("asr", model_id, asr_params), + provenance=ctx["prov"]("asr", model_id, asr_params), + **extra_kwargs, + ) + summary["asr"]["by_model"][model_id] = outcome + write_json(pass_dir / "asr" / f"{_safe(model_id)}.json", outcome) + + +def _stage_alignment(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """Auto-align text-only ASR outputs (Qwen aligner default, MMS fallback) (T012 stage). + + The alignment cache is independent from the ASR cache (FR-024); failed + alignments preserve the ASR text but fall back to a single full-audio + TextArea region in the LS export (FR-025). + """ + summary, pass_dir = ctx["summary"], ctx["pass_dir"] + audio_sig, wrapper_hash, senselab_ver = ctx["audio_sig"], ctx["wrapper_hash"], ctx["senselab_ver"] + summary["alignment"] = {"by_model": {}} + align_language = Language(language_code=args.asr_language) if args.asr_language else Language(language_code="en") + # Aligner backend for text-only ASR: Qwen3-ForcedAligner (default) or MMS. + if args.aligner == "qwen": + aligner_fn: Any = QwenASR.align_with_qwen + aligner_model_id = args.qwen_aligner_model + else: + aligner_fn = align_transcriptions + aligner_model_id = args.aligner_model + for model_id, asr_outcome in summary["asr"]["by_model"].items(): + if asr_outcome.get("status") != "ok": + continue + asr_result = asr_outcome.get("result") + if _asr_has_timestamps(asr_result): + # Already has native timestamps — alignment would be a no-op. + continue + transcript_text = _extract_transcript_text(asr_result) + if not transcript_text: + continue + transcript_sha = transcript_signature(transcript_text) + aligner_params = { + "language": align_language.language_code, + "romanize": align_language.language_code in ("ja", "zh"), + # Levels-to-keep is part of the cache key — bumping its value + # invalidates earlier entries that were stored with the all-False + # default (which produced empty chunks). + "levels_to_keep": "utterance+word", + } + align_provenance = { + **ctx["prov"]("alignment", aligner_model_id, aligner_params), + "transcript_sha": transcript_sha, + "language": align_language.language_code, + "aligner_backend": args.aligner, + "parent_asr_cache_key": asr_outcome.get("cache_key"), + } + align_key = align_cache_key( + audio_sig=audio_sig, + transcript_sha=transcript_sha, + language=align_language.language_code, + aligner_model_id=aligner_model_id, + aligner_params=aligner_params, + wrapper_hash=wrapper_hash, + senselab_ver=senselab_ver, + ) + outcome = run_alignment_cached( + f"alignment[{model_id}]", + aligner_fn, + [(audio, ScriptLine(text=transcript_text), align_language)], + # Keep word-level chunks (and the utterance wrapper) so the comparator + # can read per-token timestamps. Default is all-False which filters + # everything out and leaves a meaningless punctuation-only ScriptLine. + levels_to_keep={"utterance": True, "word": True, "char": False}, + aligner_model=aligner_model_id, + cache_dir=ctx["cache_dir"], + cache_key_str=align_key, + provenance=align_provenance, + ) + summary["alignment"]["by_model"][model_id] = outcome + write_json(pass_dir / "alignment" / f"{_safe(model_id)}.json", outcome) + + +def _stage_ppg(audio: Audio, args: argparse.Namespace, ctx: dict[str, Any]) -> None: + """PPG extraction + argmax sidecar (T012 stage).""" + summary, pass_dir, device = ctx["summary"], ctx["pass_dir"], ctx["device"] + from senselab.audio.tasks.features_extraction.ppg import ( + _PHONEME_LABELS as _PPG_PHONEME_LABELS, + ) + from senselab.audio.tasks.features_extraction.ppg import ( + extract_ppgs_from_audios, + ) + + params = {"device": ctx["device_label"]} + outcome = run_task_cached( + "ppgs", + extract_ppgs_from_audios, + [audio], + device=device, + cache_dir=ctx["cache_dir"], + cache_key_str=ctx["key"]("ppgs", "ppgs/0.0.9", params), + provenance=ctx["prov"]("ppgs", "ppgs/0.0.9", params), + ) + # Attach the phoneme inventory so the workflow's harvester can decode + # argmax indices without re-importing the ppgs library. + outcome["phoneme_labels"] = list(_PPG_PHONEME_LABELS) + summary["ppgs"] = outcome + # Emit a small sidecar JSON: the full (40 × N_frames) tensor is too + # large to dump, but the argmax-per-frame sequence + frame_hop is what + # the comparator actually consumes. Write it so reviewers can inspect + # the phoneme timeline without rerunning the PPG model. + from senselab.audio.workflows.audio_analysis.harvesters import ppg_argmax_per_frame + + argmax_payload: dict[str, Any] = { + "phoneme_labels": list(_PPG_PHONEME_LABELS), + "per_frame_phonemes": [], + "frame_hop_s": 0.0, + } + if outcome.get("status") == "ok": + try: + pf, fh = ppg_argmax_per_frame( + outcome.get("result"), + list(_PPG_PHONEME_LABELS), + audio.waveform.shape[-1] / audio.sampling_rate, + ) + argmax_payload["per_frame_phonemes"] = pf + argmax_payload["frame_hop_s"] = float(fh) + except Exception as exc: # noqa: BLE001 + argmax_payload["argmax_error"] = repr(exc) + write_json( + pass_dir / "ppgs.json", + { + **{k: v for k, v in outcome.items() if k != "result"}, + "result_summary": "argmax-per-frame sequence in 'argmax' field; full tensor in process memory only", + "argmax": argmax_payload, + }, + ) + + def run_pass( label: str, audio: Audio, @@ -1465,8 +1887,10 @@ def run_pass( ) -> dict[str, Any]: """Run all six tasks against one audio variant and persist their outputs. - Each task call is gated through the content-addressable cache: the cache - key is ``sha256(audio_signature, task, model_id, params, wrapper_hash, + Decomposed into per-stage functions (spec T012) — task names, params, and + therefore cache keys are byte-identical to the pre-decomposition inline + blocks. Each task call is gated through the content-addressable cache: the + cache key is ``sha256(audio_signature, task, model_id, params, wrapper_hash, senselab_ver)``. On cache hit the prior outcome is replayed verbatim and no model is loaded. On miss the task runs and the outcome is stored under the cache dir for future replay. @@ -1508,263 +1932,107 @@ def _key(task: str, model_id: str | None, params: dict[str, Any]) -> str: senselab_ver=senselab_ver, ) - if "diarization" not in args.skip: - summary["diarization"] = {"by_model": {}} - for model_id in args.diarization_models: - params = {"device": device_label_for_provenance} - outcome = run_task_cached( - f"diarization[{model_id}]", - diarize_audios, - [audio], - model=pick_dispatch_model(model_id, task="diarization"), - device=device, - cache_dir=cache_dir, - cache_key_str=_key("diarization", model_id, params), - provenance=_provenance_for("diarization", model_id, params), - ) - summary["diarization"]["by_model"][model_id] = outcome - write_json(pass_dir / "diarization" / f"{_safe(model_id)}.json", outcome) - - ast_win = args.ast_win_length - ast_hop = args.ast_hop_length - yam_win = args.yamnet_win_length - yam_hop = args.yamnet_hop_length - - if "ast" not in args.skip: - params = {"win_length": ast_win, "hop_length": ast_hop, "device": device_label_for_provenance} - summary["ast"] = run_task_cached( - "ast", - classify_audios, - [audio], - model=HFModel(path_or_uri=args.ast_model), - device=device, - win_length=ast_win, - hop_length=ast_hop, - cache_dir=cache_dir, - cache_key_str=_key("ast", args.ast_model, params), - provenance=_provenance_for("ast", args.ast_model, params), - ) - summary["ast"]["window"] = {"win_length": ast_win, "hop_length": ast_hop} - write_json(pass_dir / "ast.json", summary["ast"]) + ctx: dict[str, Any] = { + "summary": summary, + "pass_dir": pass_dir, + "device": device, + "device_label": device_label_for_provenance, + "cache_dir": cache_dir, + "key": _key, + "prov": _provenance_for, + "audio_sig": audio_sig, + "wrapper_hash": wrapper_hash, + "senselab_ver": senselab_ver, + } - if "yamnet" not in args.skip: - # YAMNet runs in senselab's TF subprocess venv (same pattern as NeMo - # Sortformer). senselab.classify_audios's `_is_yamnet()` dispatcher - # matches on the raw model-id *string*, not on a SenselabModel wrapper — - # passing HFModel here would fail validation (yamnet isn't on HF). - params = {"win_length": yam_win, "hop_length": yam_hop} - summary["yamnet"] = run_task_cached( - "yamnet", - classify_audios, - [audio], - model=args.yamnet_model, - win_length=yam_win, - hop_length=yam_hop, - cache_dir=cache_dir, - cache_key_str=_key("yamnet", args.yamnet_model, params), - provenance=_provenance_for("yamnet", args.yamnet_model, params), - ) - summary["yamnet"]["window"] = {"win_length": yam_win, "hop_length": yam_hop} - write_json(pass_dir / "yamnet.json", summary["yamnet"]) + if "diarization" not in args.skip: + _stage_diarization(audio, args, ctx) - # If both ran on the same grid, emit a side-by-side comparison. - if ( - "ast" not in args.skip - and "yamnet" not in args.skip - and summary.get("ast", {}).get("status") == "ok" - and summary.get("yamnet", {}).get("status") == "ok" - and ast_win == yam_win - and ast_hop == yam_hop - ): - agreement = _scene_agreement( - ast_result=summary["ast"]["result"], - yamnet_result=summary["yamnet"]["result"], - win_length=ast_win, - hop_length=ast_hop, - ) - summary["scene_agreement"] = agreement - write_json(pass_dir / "scene_agreement.json", agreement) + # AST + YAMNet + same-grid agreement (handles per-classifier skips internally). + _stage_scene(audio, args, ctx) if "features" not in args.skip: - feat_params: dict[str, Any] = { - "opensmile": "LowLevelDescriptors@native", - "parselmouth": "windowed", - "torchaudio_squim": "windowed", - "device": device_label_for_provenance, - "win_length": args.features_win_length, - "hop_length": args.features_hop_length, - } - summary["features"] = run_task_cached( - "features", - extract_temporal_features, - audio, - win_length=args.features_win_length, - hop_length=args.features_hop_length, - device=device, - cache_dir=cache_dir, - cache_key_str=_key("features", None, feat_params), - provenance=_provenance_for("features", None, feat_params), - ) - # Each backend writes its own parquet sidecar — they have - # different columns and different time grids (opensmile LLD is - # native ~10 ms; parselmouth/torchaudio_squim follow the - # ``--features-*-length`` window). Cache outcome metadata stays - # in JSON for inspection parity with the other tasks. - result = summary["features"].get("result") or {} - if isinstance(result, dict): - try: - import pandas as pd - - feat_dir = pass_dir / "features" - feat_dir.mkdir(parents=True, exist_ok=True) - for backend, rows in result.items(): - if not rows: - continue - pd.DataFrame(rows).to_parquet(feat_dir / f"{backend}.parquet", index=False) - except Exception as exc: # noqa: BLE001 — best-effort sidecar - print(f" [features] warn: parquet write failed: {exc!r}", file=sys.stderr) - write_json(pass_dir / "features.json", {**summary["features"], "result": "see features/*.parquet"}) + _stage_features(audio, args, ctx) if "asr" not in args.skip: - summary["asr"] = {"by_model": {}} - for model_id in args.asr_models: - asr_params: dict[str, Any] = {"device": device_label_for_provenance} - extra_kwargs: dict[str, Any] = {} - # Qwen3-ASR ships its own forced-aligner companion model; allow - # opt-out so the script's MMS auto-align stage can take over. - if model_id.startswith("Qwen/Qwen3-ASR") and args.qwen_asr_no_timestamps: - extra_kwargs["return_timestamps"] = False - asr_params["return_timestamps"] = False - outcome = run_task_cached( - f"asr[{model_id}]", - transcribe_audios, - [audio], - model=pick_dispatch_model(model_id, task="asr"), - device=device, - cache_dir=cache_dir, - cache_key_str=_key("asr", model_id, asr_params), - provenance=_provenance_for("asr", model_id, asr_params), - **extra_kwargs, - ) - summary["asr"]["by_model"][model_id] = outcome - write_json(pass_dir / "asr" / f"{_safe(model_id)}.json", outcome) + _stage_asr(audio, args, ctx) # Auto-align stage. Iterate every successful ASR ModelRun; when the ASR # result lacks native timestamps (Granite Speech, Canary-Qwen, etc.) and # the user hasn't disabled alignment, post-process the text through - # senselab's forced_alignment to produce per-segment timestamps. The - # alignment cache is independent from the ASR cache (FR-024); failed - # alignments preserve the ASR text but fall back to a single full-audio - # TextArea region in the LS export (FR-025). + # senselab's forced_alignment to produce per-segment timestamps. if "asr" not in args.skip and "alignment" not in args.skip and not args.no_align_asr and "asr" in summary: - summary["alignment"] = {"by_model": {}} - align_language = ( - Language(language_code=args.asr_language) if args.asr_language else Language(language_code="en") - ) - for model_id, asr_outcome in summary["asr"]["by_model"].items(): - if asr_outcome.get("status") != "ok": - continue - asr_result = asr_outcome.get("result") - if _asr_has_timestamps(asr_result): - # Already has native timestamps — alignment would be a no-op. - continue - transcript_text = _extract_transcript_text(asr_result) - if not transcript_text: - continue - transcript_sha = transcript_signature(transcript_text) - aligner_params = { - "language": align_language.language_code, - "romanize": align_language.language_code in ("ja", "zh"), - # Levels-to-keep is part of the cache key — bumping its value - # invalidates earlier entries that were stored with the all-False - # default (which produced empty chunks). - "levels_to_keep": "utterance+word", - } - align_provenance = { - **_provenance_for("alignment", args.aligner_model, aligner_params), - "transcript_sha": transcript_sha, - "language": align_language.language_code, - "parent_asr_cache_key": asr_outcome.get("cache_key"), - } - align_key = align_cache_key( - audio_sig=audio_sig, - transcript_sha=transcript_sha, - language=align_language.language_code, - aligner_model_id=args.aligner_model, - aligner_params=aligner_params, - wrapper_hash=wrapper_hash, - senselab_ver=senselab_ver, - ) - outcome = run_alignment_cached( - f"alignment[{model_id}]", - align_transcriptions, - [(audio, ScriptLine(text=transcript_text), align_language)], - # Keep word-level chunks (and the utterance wrapper) so the comparator - # can read per-token timestamps. Default is all-False which filters - # everything out and leaves a meaningless punctuation-only ScriptLine. - levels_to_keep={"utterance": True, "word": True, "char": False}, - aligner_model=args.aligner_model, - cache_dir=cache_dir, - cache_key_str=align_key, - provenance=align_provenance, - ) - summary["alignment"]["by_model"][model_id] = outcome - write_json(pass_dir / "alignment" / f"{_safe(model_id)}.json", outcome) + _stage_alignment(audio, args, ctx) if args.ppg: - from senselab.audio.tasks.features_extraction.ppg import ( - _PHONEME_LABELS as _PPG_PHONEME_LABELS, - ) - from senselab.audio.tasks.features_extraction.ppg import ( - extract_ppgs_from_audios, - ) + _stage_ppg(audio, args, ctx) - params = {"device": device_label_for_provenance} - outcome = run_task_cached( - "ppgs", - extract_ppgs_from_audios, - [audio], - device=device, - cache_dir=cache_dir, - cache_key_str=_key("ppgs", "ppgs/0.0.9", params), - provenance=_provenance_for("ppgs", "ppgs/0.0.9", params), + return summary + + +def run_triage(audio: Audio, args: argparse.Namespace, device: DeviceType | None) -> dict[str, Any]: + """Round 0 (spec US1): frame-posterior speech gate + SNR enhancement gate. + + Uses continuous segmentation-3.0 frame posteriors (never segmentized VAD — + see SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md) and Brouhaha SNR with an ungated + percentile-DSP fallback. Degrades conservatively: missing posteriors ⇒ + ``speech_present=True``; missing SNR ⇒ ``needs_enhancement=None`` (the + caller treats unknown as "run the enhanced pass"). + """ + from senselab.audio.tasks.voice_activity_detection.frame_posteriors import ( + extract_speech_frame_posteriors, + ) + from senselab.audio.workflows.audio_analysis.adaptive.triage import dsp_snr_series, triage_decision + + t0 = time.time() + posterior = extract_speech_frame_posteriors([audio], device=device)[0] + + snr_db: list[float] | None = None + snr_hop_s: float | None = None + snr_estimator: str | None = None + try: + from senselab.audio.tasks.scene_quality.brouhaha import extract_brouhaha_frames + + brouhaha = extract_brouhaha_frames([audio], device=device)[0] + if brouhaha is not None: + snr_db = [float(v) for v in brouhaha.snr_db] + snr_hop_s = float(brouhaha.frame_hop_s) + snr_estimator = "brouhaha" + except Exception as exc: # noqa: BLE001 — SNR is optional; DSP fallback below + print(f" [triage] brouhaha unavailable ({exc!r}); falling back to DSP SNR", file=sys.stderr) + if snr_db is None: + waveform = audio.waveform.squeeze().detach().cpu().numpy() + snr_db, snr_hop_s = dsp_snr_series( + waveform, + audio.sampling_rate, + p_speech=[float(p) for p in posterior.probs] if posterior is not None else None, + p_hop_s=float(posterior.frame_hop_s) if posterior is not None else None, ) - # Attach the phoneme inventory so the workflow's harvester can decode - # argmax indices without re-importing the ppgs library. - outcome["phoneme_labels"] = list(_PPG_PHONEME_LABELS) - summary["ppgs"] = outcome - # Emit a small sidecar JSON: the full (40 × N_frames) tensor is too - # large to dump, but the argmax-per-frame sequence + frame_hop is what - # the comparator actually consumes. Write it so reviewers can inspect - # the phoneme timeline without rerunning the PPG model. - from senselab.audio.workflows.audio_analysis.harvesters import ppg_argmax_per_frame - - argmax_payload: dict[str, Any] = { - "phoneme_labels": list(_PPG_PHONEME_LABELS), - "per_frame_phonemes": [], - "frame_hop_s": 0.0, + snr_estimator = "dsp_posterior_masked" if posterior is not None else "dsp_percentile" + + if posterior is None: + decision: dict[str, Any] = { + "speech_present": True, + "needs_enhancement": None, + "inconclusive": True, + "reason": "frame_posteriors_unavailable", + "stats": {}, + "thresholds": {}, } - if outcome.get("status") == "ok": - try: - pf, fh = ppg_argmax_per_frame( - outcome.get("result"), - list(_PPG_PHONEME_LABELS), - audio.waveform.shape[-1] / audio.sampling_rate, - ) - argmax_payload["per_frame_phonemes"] = pf - argmax_payload["frame_hop_s"] = float(fh) - except Exception as exc: # noqa: BLE001 - argmax_payload["argmax_error"] = repr(exc) - write_json( - pass_dir / "ppgs.json", - { - **{k: v for k, v in outcome.items() if k != "result"}, - "result_summary": "argmax-per-frame sequence in 'argmax' field; full tensor in process memory only", - "argmax": argmax_payload, - }, + else: + decision = triage_decision( + p_speech=[float(p) for p in posterior.probs], + frame_hop_s=float(posterior.frame_hop_s), + snr_db=snr_db, + snr_hop_s=snr_hop_s, + speech_threshold=args.triage_speech_threshold, + min_speech_s=args.triage_min_speech_s, + snr_floor_db=args.triage_snr_floor_db, + low_snr_fraction_threshold=args.triage_low_snr_fraction, ) - - return summary + decision["snr_estimator"] = snr_estimator + decision["elapsed_s"] = round(time.time() - t0, 3) + return decision def main(argv: list[str] | None = None) -> int: @@ -1821,6 +2089,33 @@ def main(argv: list[str] | None = None) -> int: "passes": {}, } + # ── Round 0: triage (spec US1; FR-002/003/004) ────────────────────── + enhancement_mode = "never" if args.no_enhancement else args.enhancement + triage: dict[str, Any] | None = None + if enhancement_mode == "auto": + print("\n=== Triage (round 0): frame posteriors + SNR ===") + triage = run_triage(audio_16k, args, device) + write_json(run_dir / "triage.json", triage) + summaries["triage"] = triage + stats = triage.get("stats") or {} + print( + f" speech_present={triage['speech_present']} " + f"(speech_s={stats.get('speech_s')}, fraction={stats.get('speech_fraction')}) " + f"needs_enhancement={triage['needs_enhancement']} " + f"(snr={triage.get('snr_estimator')}, median_snr={stats.get('median_snr_db_in_speech')} dB)" + ) + if not triage["speech_present"]: + summaries["run_state"] = "no_speech" + args.skip = tuple(sorted(set(args.skip) | {"diarization", "asr", "alignment"})) + args.ppg = False + print(" no speech found — skipping diarization/ASR/alignment/PPG; presence outputs still emitted (FR-004)") + run_enhanced_pass = enhancement_mode == "always" or ( + enhancement_mode == "auto" + and triage is not None + and triage["speech_present"] + and triage["needs_enhancement"] is not False # unknown SNR ⇒ conservative: run it + ) + pass_audio: dict[str, Audio] = {"raw_16k": audio_16k} summaries["passes"]["raw_16k"] = run_pass( @@ -1834,7 +2129,7 @@ def main(argv: list[str] | None = None) -> int: senselab_ver=senselab_ver, ) - if not args.no_enhancement: + if run_enhanced_pass: print("\n=== Enhancing audio (this loads the enhancement model)... ===") try: enhanced = enhance_audios( @@ -1898,18 +2193,43 @@ def main(argv: list[str] | None = None) -> int: win_length=args.utterance_win_length, hop_length=args.utterance_hop_length, ) + presence_grid = BucketGrid( + win_length=args.presence_grid_win_length, + hop_length=args.presence_grid_hop_length, + ) comparator_params = { "win_length": grid.win_length, "hop_length": grid.hop_length, "utterance_win_length": utterance_grid.win_length, "utterance_hop_length": utterance_grid.hop_length, + "presence_win_length": presence_grid.win_length, + "presence_hop_length": presence_grid.hop_length, "aggregator": args.uncertainty_aggregator, "phoneme_disagreement_threshold": args.phoneme_disagreement_threshold, "speech_presence_labels": _speech_presence_labels(args), "asr_reference_model": args.asr_reference_model, "diarization_boundary_shift_ms": args.diarization_boundary_shift_ms, "clustering_algorithm": args.clustering_algorithm, + "utterance_scene_coupling": { + "w_q": float(args.utterance_scene_coupling_weights[0]), + "w_s": float(args.utterance_scene_coupling_weights[1]), + }, } + # US5 (T039): load + validate the calibration profile and thread its flat + # runtime form into BOTH consumers — the harvest (quality dB→[0,1] anchors, + # via the calibration kwarg below) and the aggregators (temperatures / + # token-entropy reference, via comparator_params["calibration"]). + from senselab.audio.workflows.audio_analysis.calibration import ( + load_calibration_profile, + profile_to_runtime, + ) + + try: + calibration_runtime = profile_to_runtime(load_calibration_profile(args.calibration_profile)) + except (OSError, ValueError, KeyError) as exc: + print(f"ERROR: invalid calibration profile {args.calibration_profile}: {exc}", file=sys.stderr) + sys.exit(2) + comparator_params["calibration"] = calibration_runtime passes_for_compute = { pl: ps for pl, ps in summaries.get("passes", {}).items() if isinstance(ps, dict) and "duration_s" in ps @@ -1926,6 +2246,10 @@ def main(argv: list[str] | None = None) -> int: aggregator=args.uncertainty_aggregator, speech_presence_labels=_speech_presence_labels(args), utterance_grid=utterance_grid, + presence_grid=presence_grid, + scene_quality=not args.no_scene_quality, + sound_sources=not args.no_sound_sources, + calibration=calibration_runtime, embedding_window_s=args.embedding_window_s, embedding_hop_s=args.embedding_hop_s, same_speaker_floor=args.identity_same_speaker_floor, @@ -1939,6 +2263,29 @@ def main(argv: list[str] | None = None) -> int: axis_results, incomparable_reasons = ({}, {"workflow": f"failed: {exc!r}"}) per_window_embeddings_by_pass = {} + # Scene quality is a REQUIRED signal here unless explicitly disabled: the + # library degrades gracefully to null (FR-023) for reuse, but this script + # must not silently ship a run missing its quality columns. If the model + # was requested but unavailable on any pass, fail loudly with guidance. + if not args.no_scene_quality: + unavailable_passes = [ + pl + for (pl, axis), result in axis_results.items() + if axis == "presence" + and pl != "raw_vs_enhanced" + and (result.provenance.get("scene_quality") or {}).get("model", {}).get("available") is False + ] + if unavailable_passes: + print( + "ERROR: scene-quality model (pyannote/brouhaha) could not be loaded for " + f"pass(es) {sorted(unavailable_passes)}, so SNR/reverb quality columns would be null. " + "Ensure the model is accessible (request access at https://hf.co/pyannote/brouhaha, " + "set HF_TOKEN) and its backend is installed, or pass --no-scene-quality to run " + "without scene quality intentionally.", + file=sys.stderr, + ) + sys.exit(2) + # Persist 9 parquets (3 axes × 2 passes + 3 raw_vs_enhanced deltas). for (pass_label, axis), result in axis_results.items(): if pass_label == "raw_vs_enhanced": diff --git a/scripts/calibrate_scene_quality.py b/scripts/calibrate_scene_quality.py new file mode 100644 index 000000000..1c7842284 --- /dev/null +++ b/scripts/calibrate_scene_quality.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Fit a scene-quality calibration profile from synthetic mixtures (US5, T037/T038). + +Synthesizes controlled degradations of a clean reference clip — additive +white/pink noise across an SNR sweep and synthetic exponential-decay RIRs across +an RT60 sweep — runs the workflow's quality estimators on each variant, fits the +dB→[0,1] normalization anchors, and persists a versioned +``CalibrationProfile`` (data-model §5) plus a reported-vs-true validation plot +and table (FR-022 / SC-007). + +Fitting scope (documented): the dB anchors (SNR always; C50 when Brouhaha is +available — the DSP path has no reverb estimator, in which case the C50 block +keeps its documented defaults and the provenance says so). The per-axis +``temperature`` entries are CLI passthroughs (default 1.0): a proper temperature +fit needs labeled correctness (e.g. the adaptive loop's ground-truth harness), +not a synthetic sweep. + +Usage (full senselab environment): + uv run python scripts/calibrate_scene_quality.py \ + --audio tutorial_audio_files/audio_48khz_mono_16bits.wav \ + --out artifacts/scene_quality_calibration.json +Then pass it to the pipeline: + uv run python scripts/analyze_audio.py clip.wav \ + --calibration-profile artifacts/scene_quality_calibration.json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """CLI (constitution VIII: every input/output/sweep is a parameter with a default).""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--audio", type=Path, default=Path("tutorial_audio_files/audio_48khz_mono_16bits.wav")) + parser.add_argument("--snr-sweep-db", type=float, nargs="+", default=[30.0, 25.0, 20.0, 15.0, 10.0, 5.0, 0.0]) + parser.add_argument("--rt60-sweep-s", type=float, nargs="+", default=[0.0, 0.3, 0.6, 0.9, 1.2]) + parser.add_argument("--noise", choices=("white", "pink"), default="white") + parser.add_argument("--clean-anchor-db", type=float, default=25.0, help="True SNR mapped to degradation 0") + parser.add_argument("--floor-anchor-db", type=float, default=5.0, help="True SNR mapped to degradation 1") + parser.add_argument("--temperature-presence", type=float, default=1.0) + parser.add_argument("--temperature-utterance", type=float, default=1.0) + parser.add_argument("--out", type=Path, default=Path("artifacts/scene_quality_calibration.json")) + parser.add_argument("--plot", type=Path, default=Path("artifacts/scene_quality_calibration_validation.png")) + parser.add_argument("--table", type=Path, default=Path("artifacts/scene_quality_calibration_validation.json")) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--no-brouhaha", action="store_true", help="Skip the Brouhaha C50 sweep (DSP-only fit)") + return parser.parse_args(argv) + + +def _synth_noise(shape: int, kind: str, rng: Any) -> Any: # noqa: ANN401 — np.ndarray + import numpy as np + + white = rng.normal(0.0, 1.0, shape) + if kind == "white": + return white + # Pink: 1/f shaping in the frequency domain. + spectrum = np.fft.rfft(white) + freqs = np.fft.rfftfreq(shape) + freqs[0] = freqs[1] if len(freqs) > 1 else 1.0 + pink = np.fft.irfft(spectrum / np.sqrt(freqs), n=shape) + return pink / max(1e-12, float(np.std(pink))) + + +def _mix_at_snr(speech: Any, noise: Any, snr_db: float) -> Any: # noqa: ANN401 + import numpy as np + + speech_power = float(np.mean(speech**2)) + noise_power = float(np.mean(noise**2)) + scale = np.sqrt(speech_power / (10 ** (snr_db / 10)) / max(1e-12, noise_power)) + return np.clip(speech + scale * noise, -1.0, 1.0).astype("float32") + + +def _exponential_rir(rt60_s: float, sr: int, rng: Any, length_s: float = 1.5) -> Any: # noqa: ANN401 + import numpy as np + + n = max(1, int(length_s * sr)) + t = np.arange(n) / sr + decay = np.exp(-6.9077 * t / max(1e-3, rt60_s)) # -60 dB at rt60 + h = decay * rng.normal(0.0, 1.0, n) + h[0] = 1.0 + return h / max(1e-12, float(np.max(np.abs(h)))) + + +def _true_c50_db(h: Any, sr: int) -> float: # noqa: ANN401 + import numpy as np + + k = int(0.05 * sr) + early = float(np.sum(h[:k] ** 2)) + late = max(1e-12, float(np.sum(h[k:] ** 2))) + return 10.0 * float(np.log10(max(1e-12, early) / late)) + + +def _estimate(audio_np: Any, sr: int, *, brouhaha: bool) -> dict[str, float | None]: # noqa: ANN401 + """Median raw estimator outputs (dB) over the clip via the workflow's quality harvest.""" + import numpy as np + import torch + + from senselab.audio.data_structures import Audio + from senselab.audio.workflows.audio_analysis.grid import BucketGrid + from senselab.audio.workflows.audio_analysis.quality import harvest_quality_scores + + audio = Audio(waveform=torch.from_numpy(audio_np).unsqueeze(0), sampling_rate=sr) + brouhaha_frames = None + if brouhaha: + from senselab.audio.tasks.scene_quality import extract_brouhaha_frames + + brouhaha_frames = extract_brouhaha_frames([audio])[0] + rows = harvest_quality_scores(audio=audio, brouhaha=brouhaha_frames, grid=BucketGrid(0.5, 0.5), calibration=None) + + def _median_raw(key: str) -> float | None: + vals = [r["_raw"][key] for r in rows if isinstance(r.get("_raw"), dict) and r["_raw"].get(key) is not None] + return float(np.median(vals)) if vals else None + + return {"snr_db": _median_raw("primary_snr_db"), "c50_db": _median_raw("brouhaha_c50_db")} + + +def _fit_anchors( + true_db: list[float], est_db: list[float], clean_anchor: float, floor_anchor: float +) -> dict[str, float]: + """Least-squares line est ≈ a·true + b → anchors = predicted est at the true targets.""" + import numpy as np + + a, b = np.polyfit(np.asarray(true_db, dtype=float), np.asarray(est_db, dtype=float), 1) + clean_db = float(a * clean_anchor + b) + floor_db = float(a * floor_anchor + b) + if clean_db <= floor_db: # estimator not monotone enough to fit — fail loudly + raise ValueError( + f"fitted anchors are inverted (clean {clean_db:.2f} <= floor {floor_db:.2f}); " + f"estimator slope a={a:.3f} — check the sweep / estimator" + ) + return {"clean_db": round(clean_db, 3), "floor_db": round(floor_db, 3), "slope": round(float(a), 4)} + + +def main(argv: list[str] | None = None) -> int: + """Run the sweeps, fit, validate, persist.""" + args = parse_args(argv) + import numpy as np + + from senselab.audio.workflows.audio_analysis.calibration import ( + DEFAULT_PROFILE, + linear_db_to_unit, + validate_profile, + ) + + rng = np.random.default_rng(args.seed) + + # Clean reference via the pipeline's own loader chain (16 kHz mono parity). + from senselab.audio.tasks.input_output import read_audios + from senselab.audio.tasks.preprocessing import downmix_audios_to_mono, resample_audios + + audio = read_audios([str(args.audio)])[0] + audio = downmix_audios_to_mono([audio])[0] + if audio.sampling_rate != 16000: + audio = resample_audios([audio], resample_rate=16000)[0] + speech = audio.waveform.detach().cpu().numpy().squeeze().astype("float32") + sr = 16000 + + # ── SNR sweep ──────────────────────────────────────────────────────── + snr_true, snr_est = [], [] + noise = _synth_noise(len(speech), args.noise, rng) + for target_db in args.snr_sweep_db: + est = _estimate(_mix_at_snr(speech, noise, target_db), sr, brouhaha=False) + if est["snr_db"] is not None: + snr_true.append(float(target_db)) + snr_est.append(float(est["snr_db"])) + print(f" SNR sweep: true={target_db:6.1f} dB → estimated={est['snr_db']}") + if len(snr_true) < 3: + print("ERROR: fewer than 3 usable SNR points — cannot fit anchors", file=sys.stderr) + return 2 + snr_fit = _fit_anchors(snr_true, snr_est, args.clean_anchor_db, args.floor_anchor_db) + + # ── RT60 / C50 sweep (Brouhaha only — the DSP path has no reverb estimator) ── + c50_fit: dict[str, float] | None = None + c50_true, c50_est = [], [] + if not args.no_brouhaha: + for rt60 in args.rt60_sweep_s: + if rt60 <= 0: + reverbed, true_c50 = speech, 40.0 # effectively anechoic + else: + h = _exponential_rir(rt60, sr, rng) + reverbed = np.convolve(speech, h)[: len(speech)].astype("float32") + peak = max(1e-9, float(np.max(np.abs(reverbed)))) + reverbed = reverbed / peak if peak > 1.0 else reverbed + true_c50 = _true_c50_db(h, sr) + est = _estimate(reverbed, sr, brouhaha=True) + print(f" RT60 sweep: rt60={rt60:4.1f} s (true C50≈{true_c50:6.1f} dB) → estimated={est['c50_db']}") + if est["c50_db"] is not None: + c50_true.append(float(true_c50)) + c50_est.append(float(est["c50_db"])) + if len(c50_true) >= 3: + c50_fit = _fit_anchors(c50_true, c50_est, clean_anchor=30.0, floor_anchor=-5.0) + else: + print("warn: Brouhaha C50 unavailable — keeping default reverb anchors", file=sys.stderr) + + # ── Profile (data-model §5) + provenance ──────────────────────────── + profile: dict[str, Any] = { + "version": "1", + "snr": {"type": "linear_db_to_unit", "clean_db": snr_fit["clean_db"], "floor_db": snr_fit["floor_db"]}, + "reverb_c50": ( + {"type": "linear_db_to_unit", "clean_db": c50_fit["clean_db"], "floor_db": c50_fit["floor_db"]} + if c50_fit + else dict(DEFAULT_PROFILE["reverb_c50"]) + ), + "bandwidth": dict(DEFAULT_PROFILE["bandwidth"]), + "temperature": {"presence": args.temperature_presence, "utterance": args.temperature_utterance}, + "provenance": { + "fitted_by": "scripts/calibrate_scene_quality.py", + "audio": args.audio.name, + "noise": args.noise, + "snr_sweep_db": list(args.snr_sweep_db), + "rt60_sweep_s": list(args.rt60_sweep_s), + "seed": args.seed, + "snr_fit": snr_fit, + "c50_fit": c50_fit or "defaults (Brouhaha unavailable or --no-brouhaha)", + "temperature_note": "CLI passthrough — fit requires labeled correctness (adaptive eval harness)", + "timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), + }, + } + validate_profile(profile, source="fitted") + args.out.parent.mkdir(parents=True, exist_ok=True) + # indent=4 + trailing newline to match the repo's pretty-format-json / + # end-of-file-fixer hooks, so a profile fitted into the package tree is + # committable without a reformat round-trip. + args.out.write_text(json.dumps(profile, indent=4) + "\n") + print(f"profile: {args.out}") + + # ── Validation table + plot (T038 / FR-022, SC-007) ────────────────── + reported = [linear_db_to_unit(e, snr_fit["clean_db"], snr_fit["floor_db"]) for e in snr_est] + monotone = all(reported[i] <= reported[i + 1] + 1e-9 for i in range(len(reported) - 1)) + table = { + "snr": [ + {"true_db": t, "estimated_db": e, "reported_degradation": round(r, 4)} + for t, e, r in zip(snr_true, snr_est, reported) + ], + "c50": [{"true_db": t, "estimated_db": e} for t, e in zip(c50_true, c50_est)], + "monotone_reported_vs_true": monotone, + } + args.table.parent.mkdir(parents=True, exist_ok=True) + args.table.write_text(json.dumps(table, indent=4) + "\n") + print(f"table: {args.table} (monotone={monotone})") + + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4)) + ax1.plot(snr_true, snr_est, "o-", label="estimated dB") + ax1.axhline(snr_fit["clean_db"], ls="--", c="green", label=f"clean anchor {snr_fit['clean_db']:.1f}") + ax1.axhline(snr_fit["floor_db"], ls="--", c="red", label=f"floor anchor {snr_fit['floor_db']:.1f}") + ax1.set_xlabel("true SNR (dB)") + ax1.set_ylabel("estimated SNR (dB)") + ax1.set_title("Estimator vs truth (SNR sweep)") + ax1.legend(fontsize=8) + ax2.plot(snr_true, reported, "s-", color="tab:red") + ax2.set_xlabel("true SNR (dB)") + ax2.set_ylabel("reported degradation [0,1]") + ax2.set_title(f"Calibrated mapping (monotone={monotone})") + ax2.invert_xaxis() + fig.tight_layout() + args.plot.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(args.plot, dpi=130) + print(f"plot: {args.plot}") + except Exception as exc: # noqa: BLE001 — plot is a best-effort sidecar + print(f"warn: validation plot failed: {exc!r}", file=sys.stderr) + + if not monotone: + print("ERROR: reported degradation is not monotone vs true SNR (SC-007)", file=sys.stderr) + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/make_degradation_suite.py b/scripts/make_degradation_suite.py new file mode 100644 index 000000000..a0b76c7a1 --- /dev/null +++ b/scripts/make_degradation_suite.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Synthetic degradation suite generator (spec T039, SC-001/006). + +Takes a clean reference clip and emits variants with *localized, known* degradations +plus a machine-readable manifest of the injected spans — the ground truth the +adaptive loop is scored against (SC-001: are injected spans proposed as regions and +either improved or correctly explained?). + +Variants (all spans recorded in manifest.json): +- ``noise``: additive white noise burst over one span +- ``clip``: hard digital clipping over one span +- ``lowpass``: telephone-band (~3.4 kHz) filtering over one span +- ``silence``: the span is zeroed (tests presence + FR-004 interactions) +- ``music``: additive synthetic "music" (chord of sines) — hallucination bait (SC-006) + +Usage: + uv run python scripts/make_degradation_suite.py tutorial_audio_files/audio_48khz_mono_16bits.wav \ + --out artifacts/degradation_suite + # then run analyze_audio.py + scripts/adaptive_loop.py on each variant and check + # SC-001 via src/tests/audio/workflows/audio_analysis/adaptive/adaptive_e2e_test.py +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main(argv: list[str] | None = None) -> int: + """Generate the suite.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("audio", type=Path) + parser.add_argument("--out", type=Path, default=Path("artifacts/degradation_suite")) + parser.add_argument( + "--span-start-fraction", type=float, default=0.4, help="Injected span start (fraction of duration)" + ) + parser.add_argument("--span-length-s", type=float, default=3.0) + parser.add_argument("--noise-std", type=float, default=0.12) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args(argv) + + import numpy as np + import soundfile as sf + + data, sr = sf.read(str(args.audio), dtype="float32", always_2d=True) + wav = data.mean(axis=1) + duration = len(wav) / sr + start = round(args.span_start_fraction * duration, 3) + end = round(min(duration, start + args.span_length_s), 3) + lo, hi = int(start * sr), int(end * sr) + rng = np.random.default_rng(args.seed) + + args.out.mkdir(parents=True, exist_ok=True) + manifest: dict[str, object] = { + "source": str(args.audio), + "sample_rate": sr, + "duration_s": round(duration, 3), + "seed": args.seed, + "variants": {}, + } + + def _emit(name: str, y: "np.ndarray", kind: str) -> None: + path = args.out / f"{args.audio.stem}__{name}.wav" + sf.write(str(path), np.clip(y, -1.0, 1.0), sr) + manifest["variants"][name] = { # type: ignore[index] + "path": str(path), + "kind": kind, + "injected_span": [start, end], + } + print(f" {name}: {path} span=[{start}, {end}]") + + noise = wav.copy() + noise[lo:hi] += rng.normal(0, args.noise_std, hi - lo).astype("float32") + _emit("noise", noise, "additive_noise") + + clip = wav.copy() + clip[lo:hi] = np.clip(clip[lo:hi] * 8.0, -0.98, 0.98) + _emit("clip", clip, "hard_clipping") + + from scipy.signal import butter, sosfilt + + sos = butter(6, 3400, btype="low", fs=sr, output="sos") + lowpass = wav.copy() + lowpass[lo:hi] = sosfilt(sos, lowpass[lo:hi]).astype("float32") + _emit("lowpass", lowpass, "band_limited") + + silence = wav.copy() + silence[lo:hi] = rng.normal(0, 1e-4, hi - lo).astype("float32") + _emit("silence", silence, "zeroed_span") + + t = np.arange(hi - lo) / sr + chord = sum(np.sin(2 * np.pi * f * t) for f in (220.0, 277.2, 329.6)) / 3.0 + music = wav.copy() + music[lo:hi] += (0.15 * chord).astype("float32") + _emit("music", music, "additive_music") + + (args.out / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(f"manifest: {args.out / 'manifest.json'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/specs/20260722-175022-scene-quality-utterance/checklists/requirements.md b/specs/20260722-175022-scene-quality-utterance/checklists/requirements.md new file mode 100644 index 000000000..81e526af1 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/checklists/requirements.md @@ -0,0 +1,36 @@ +# Specification Quality Checklist: Scene-aware presence axis + improved utterance uncertainty + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-22 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- The spec necessarily names concrete signals/models the user explicitly requested (SNR, clipping, reverb, bandwidth; `pyannote/brouhaha`; AST/YAMNet) because they *are* the requirements. These are named as the WHAT (which signals must exist), not as prescribed code structure; the HOW is deferred to planning. This is consistent with prior specs in this repo (e.g. the auditory-scene-analysis spec names YAMNet). +- No [NEEDS CLARIFICATION] markers: the four upfront design decisions (rework presence vs new axis; quality signals; source-model appetite; utterance fixes) and three follow-ups (output shape; scope; calibration source) were resolved with the user during brainstorming before this spec was written. +- Items marked incomplete would require spec updates before `/speckit.clarify` or `/speckit.plan`; none are currently incomplete. diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/cli.md b/specs/20260722-175022-scene-quality-utterance/contracts/cli.md new file mode 100644 index 000000000..257618d1c --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/cli.md @@ -0,0 +1,43 @@ +# Contract: CLI additions (`scripts/analyze_audio.py`) + +All new flags have defaults so the common case needs zero configuration (constitution VIII). + +| Flag | Default | Effect | +|---|---|---| +| `--presence-grid-win` / `--presence-grid-hop` | `0.1` / `0.02` | presence reporting grid (new `presence_grid`) | +| `--scene-top-k` | full (527 AST / 521 YAMNet) | class count persisted from AST/YAMNet windowed classification (enables source masses) | +| `--scene-model` | `pyannote/brouhaha` | scene/quality model id | +| `--no-scene-quality` | off | skip Brouhaha + quality columns (columns emitted null) | +| `--no-sound-sources` | off | skip source categorization (columns null) | +| `--calibration-profile` | bundled default | path to a fitted `CalibrationProfile` JSON | +| `--utterance-scene-coupling-weights` | documented defaults `w_q`, `w_s` | scene→utterance coupling strength | + +## `compute_uncertainty_axes` signature additions + +```python +def compute_uncertainty_axes( + *, + passes, grid, params, audio, + utterance_grid: BucketGrid | None = None, # existing + presence_grid: BucketGrid | None = None, # NEW (default 0.1/0.02) + scene_quality: bool = True, # NEW + sound_sources: bool = True, # NEW + calibration: CalibrationProfile | None = None, # NEW + ... +) -> tuple[list[AxisResult], list]: + ... +``` + +- Defaults preserve current behavior for callers that don't pass the new kwargs, except that presence now reports on the finer default grid; a caller can pass `presence_grid=grid` to retain the old 0.5 s presence resolution (documented in quickstart). + +## Calibration helper — `scripts/calibrate_scene_quality.py` + +``` +uv run python scripts/calibrate_scene_quality.py \ + --clean tutorial_audio_files/.wav \ + --snr-sweep 30 20 10 5 0 \ + --rt60-sweep 0.2 0.5 1.0 \ + --out src/senselab/audio/workflows/audio_analysis/data/scene_quality_calibration.json +``` + +- Synthesizes noise + decay-RIR mixtures (numpy, no new dep), runs the quality estimators, fits normalization + temperature, writes the profile, and emits a reported-vs-true validation plot under `artifacts/`. diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/frame_posteriors.md b/specs/20260722-175022-scene-quality-utterance/contracts/frame_posteriors.md new file mode 100644 index 000000000..575124641 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/frame_posteriors.md @@ -0,0 +1,49 @@ +# Contract: frame-level speech posteriors + +**Module**: `src/senselab/audio/tasks/voice_activity_detection/frame_posteriors.py` (new) + +## Public function + +```python +def extract_speech_frame_posteriors( + audios: list[Audio], + model: PyannoteAudioModel | None = None, # default pyannote/segmentation-3.0, pinned revision + device: DeviceType | None = None, +) -> list[FramePosterior]: + ... +``` + +Where `FramePosterior` is a small dataclass: + +```python +@dataclass +class FramePosterior: + probs: np.ndarray # shape (num_frames,), P(speech) in [0,1] + frame_hop_s: float # seconds per frame (~0.0169 for segmentation-3.0) + frame_win_s: float # analysis window per frame +``` + +## Behavior + +- Loads the model via `pyannote.audio.Model.from_pretrained(repo, revision=..., token=get_huggingface_token())` wrapped by `ensure_hf_model` + `local_files_only` when cached (constitution VI). Cached per `(repo, revision, device)`. +- Runs `pyannote.audio.Inference(model, ...)` → `SlidingWindowFeature`; collapses the class/powerset axis with `max` → continuous P(speech) per frame. +- **Does NOT** route through `VoiceActivityDetection`/`Pipeline` (that re-segments and smooths). +- Returns per-frame probability arrays, not `ScriptLine` segments. + +## Bucket aggregation helper + +```python +def mean_posterior_in_window(fp: FramePosterior, start_s: float, end_s: float) -> tuple[float, float]: + """Return (mean P(speech), within-window frame std) over frames overlapping [start,end).""" +``` + +- Mean → presence confidence contribution; std → within-bucket temporal-instability contribution to `presence_uncertainty`. +- Empty overlap → `(nan, nan)` handled by caller as a missing voter. + +## Error handling + +- Model unavailable (no token / not cached / import failure `(ImportError, RuntimeError)`) → function returns posteriors from whatever backends loaded; a fully failed model contributes no voter (FR-023). Never aborts the workflow. + +## Brouhaha variant + +`scene_quality/brouhaha.py` exposes the analogous multitask extractor (see `contracts/quality.md`); its VAD head reuses `FramePosterior` so presence gets a second frame-posterior voter with identical bucket aggregation. diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/presence-parquet-columns.md b/specs/20260722-175022-scene-quality-utterance/contracts/presence-parquet-columns.md new file mode 100644 index 000000000..bbf5ba7f1 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/presence-parquet-columns.md @@ -0,0 +1,36 @@ +# Contract: presence parquet — additive columns + +**Writer**: `src/senselab/audio/workflows/audio_analysis/io.py::write_axis_parquet` + +Existing columns (unchanged): `start, end, axis, aggregated_uncertainty, raw_aggregated_uncertainty, intensity_weight, contributing_models (list), model_votes (JSON string), comparison_status`. + +## New columns (appended) + +| Column | pyarrow type | Axis populated | Null when | +|---|---|---|---| +| `presence_confidence` | `float64` | presence | no votes | +| `presence_uncertainty` | `float64` | presence | no votes | +| `quality_snr` | `float64` | presence | Brouhaha+DSP SNR all unavailable | +| `quality_clip` | `float64` | presence | — (cheap, always available) | +| `quality_reverb` | `float64` | presence | Brouhaha unavailable | +| `quality_bandwidth` | `float64` | presence | — | +| `quality_uncertainty` | `float64` | presence | <2 SNR estimators available | +| `src_speech` | `float64` | presence | AST+YAMNet absent | +| `src_people` | `float64` | presence | AST+YAMNet absent | +| `src_machine` | `float64` | presence | AST+YAMNet absent | +| `src_environment` | `float64` | presence | AST+YAMNet absent | +| `src_dominant` | `string` | presence | AST+YAMNet absent | +| `token_entropy` | `float64` | utterance | no token-scoring backend | +| `scene_quality_coupling` | `float64` | utterance | no quality span overlap | + +## Guarantees + +- On the `identity` axis and (for presence-only columns) the `utterance` axis, the columns exist but are all-null — one uniform schema across the three parquets keeps readers simple. +- **Backward compatibility (SC-008)**: existing columns keep identical names, types, order, and values; `model_votes` JSON gains new per-vote keys (`quality_*`, `src_*`, frame-posterior fields) with no schema change. Column-projecting readers (LS bundle, plot, disagreements) ignore the new columns unless updated. +- Provenance (`schema.metadata[b"comparator_provenance"]`) gains: per-axis grid params, quality analysis-window params, category-map version, calibration-profile version, and model ids/revisions (Brouhaha, segmentation-3.0) — per FR-015 and memory `project_model_revision_pinning`. + +## Disagreements / LS / plot (FR-024) + +- `disagreements.py`: new presence sub-signals become rankable entries (quality/source spikes surface alongside the existing axis disagreements). +- `labelstudio.py`: add `__presence__quality` and `__presence__sources` tracks (additive; existing `presence` track unchanged). +- `plot.py`: optional extra rows for quality and dominant-source strips; existing 5 rows unchanged when new signals are null. diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/quality.md b/specs/20260722-175022-scene-quality-utterance/contracts/quality.md new file mode 100644 index 000000000..59067734b --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/quality.md @@ -0,0 +1,64 @@ +# Contract: scene-quality model + per-bucket quality harvester + +## A. Brouhaha loader — `scene_quality/brouhaha.py` (new) + +```python +@dataclass +class BrouhahaFrames: + vad: np.ndarray # (num_frames,) P(speech) in [0,1] + snr_db: np.ndarray # (num_frames,) estimated SNR in dB + c50_db: np.ndarray # (num_frames,) estimated C50 in dB + frame_hop_s: float + +def extract_brouhaha_frames( + audios: list[Audio], + model_id: str = "pyannote/brouhaha", + revision: str = "", + device: DeviceType | None = None, +) -> list[BrouhahaFrames | None]: + ... +``` + +- Loads via `Model.from_pretrained` + `Inference` (same pattern/caching as `frame_posteriors.py`), through `ensure_hf_model` + `local_files_only`, gated `HF_TOKEN`. No new pip dependency, no subprocess venv (FR-025). +- One forward pass on the whole 16 kHz mono audio → three per-frame arrays. +- Returns `None` per audio when the model can't load (FR-023). + +## B. Quality harvester — `workflows/audio_analysis/quality.py` (new) + +```python +QUALITY_ANALYSIS_WIN_S = 0.5 +QUALITY_ANALYSIS_HOP_S = 0.25 + +def harvest_quality_scores( + *, + audio: Audio, # raw 16 kHz mono + brouhaha: BrouhahaFrames | None, + grid: BucketGrid, # the presence reporting grid + calibration: CalibrationProfile | None = None, +) -> list[dict[str, Any]]: + """One dict per presence bucket: {'start','end', quality_snr, quality_clip, + quality_reverb, quality_bandwidth, quality_uncertainty, '_raw': {...}}.""" +``` + +### Signal computation (all → `[0,1]` degradation, 0 = clean) + +| Column | Source | Normalization | +|---|---|---| +| `quality_snr` | Brouhaha `snr_db` mean-in-analysis-window (primary); cross-check vs `spectral_gating_snr_metric`, `peak_snr_from_spectral_metric` on the 0.5 s slice | `clip((clean_db − snr_db)/(clean_db − floor_db), 0, 1)` per calibration | +| `quality_clip` | `proportion_clipped_metric` on the bucket slice (cheap, slice-safe) | already `[0,1]`; optional severity curve | +| `quality_reverb` | Brouhaha `c50_db` mean-in-window | `clip((clean_db − c50_db)/(clean_db − floor_db), 0, 1)` | +| `quality_bandwidth` | `librosa.feature.spectral_rolloff(y, sr, roll_percent=0.85)` mean over analysis window | `clip(1 − rolloff_hz / nyquist_ref_hz, 0, 1)` | +| `quality_uncertainty` | normalized stddev across the ≥2 independent SNR estimators (Brouhaha, DSP-a, DSP-b) mapped to `[0,1]` | high when estimators disagree | + +### Rules + +- **Analysis resolution ≠ reporting grid**: SNR/reverb/bandwidth computed on the fixed 0.5 s / 0.25 s analysis window; each presence bucket takes the value of its containing/nearest analysis window (broadcast). `quality_clip` may be computed on the bucket slice directly. The 0.5 s analysis grid is recorded in provenance. +- **Slicing** follows `embeddings.py:_slice_audio` + tail-anchored `_window_starts` so short trailing buckets get a valid-length slice; STFT metrics need ≥~128 ms. +- **`voice_signal_to_noise_power_ratio_metric` is NOT used per bucket** (internal VAD). +- **SQUIM** (`extract_objective_quality_features_from_audios`) is an optional secondary cross-check for `quality_snr`, batched over analysis windows, gated on `torchaudio_available()`; omitted from the P1 minimum. +- Any missing source → that column is `None` for the affected buckets (FR-023), rest still emitted. +- Raw dB/estimator values retained under `_raw` → serialized into `model_votes` JSON, not columns. + +## C. librosa dependency + +`librosa` promoted from transitive to an explicit `pyproject.toml` dependency (D8). diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/sound_sources.md b/specs/20260722-175022-scene-quality-utterance/contracts/sound_sources.md new file mode 100644 index 000000000..4a960780a --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/sound_sources.md @@ -0,0 +1,47 @@ +# Contract: sound-source categorization + +## A. Category map — `workflows/audio_analysis/data/audioset_source_map.json` (new) + +Schema in `data-model.md §3`. Categories `{speech, people, machine, environment}`, default `environment`, keys = AudioSet display names covering AST (527) ∪ YAMNet (521). + +**Invariant (SC-003)**: every emittable class maps to exactly one category. Enforced by test (see D). + +## B. Enabling full distributions (mandatory prerequisite) + +Today `_classify_windowed` truncates to `top_k=5`. The AST + YAMNet call sites in `scripts/analyze_audio.py` must pass the full label count: + +```python +classify_audios(..., win_length=w, hop_length=h, top_k=scene_top_k) # scene_top_k default = full (527 / 521) +``` + +- New CLI param `--scene-top-k` (default: full per model). Top-1 consumers (`speech_presence_labels`, YAMNet veto) are unaffected — they read `labels[0]`. +- Stored per-window dict is unchanged in shape (`labels`/`scores` parallel arrays), just longer. + +## C. Harvester — `workflows/audio_analysis/sound_sources.py` (new) + +```python +def harvest_source_categories( + *, + pass_summary: dict[str, Any], # reads pass_summary['ast'], pass_summary['yamnet'] + grid: BucketGrid, # presence reporting grid + category_map: SoundSourceCategoryMap, +) -> list[dict[str, Any]]: + """One dict per presence bucket: + {'start','end', src_speech, src_people, src_machine, src_environment, + src_dominant, '_raw': {per-category class contributions}}.""" +``` + +### Rules + +- For each classifier window overlapping a bucket, sum `scores` into category masses via `category_map`; combine AST + YAMNet (mean of available; YAMNet-authoritative tie-break consistent with existing `_speech_window_mask` policy is **not** required here — masses are additive, not vetoed). +- Normalize the four masses to sum ≈ 1 per bucket; `src_dominant = argmax`. +- Project classifier windows onto buckets with the existing center→nearest-window logic (`presence.py:335`). +- Both classifiers absent → all `src_*` = `None` (FR-023). +- Unmapped class → `default` category + one logged warning per unique class (not per occurrence). + +## D. Coverage test (SC-003) + +`sound_sources_test.py::test_category_map_covers_all_classes`: +- Load AST `id2label` (`AutoConfig.from_pretrained(AST_ID).id2label`) and the YAMNet class list (vendored copy or model asset). +- Assert `set(classes) ⊆ set(map.keys())` OR every class resolves (mapped or default) with **zero** silent gaps, and each maps to exactly one of the 4 categories. +- Assert masses from a synthetic window sum to 1. diff --git a/specs/20260722-175022-scene-quality-utterance/contracts/utterance.md b/specs/20260722-175022-scene-quality-utterance/contracts/utterance.md new file mode 100644 index 000000000..7f47a1593 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/contracts/utterance.md @@ -0,0 +1,63 @@ +# Contract: utterance estimator improvements + +## A. Whisper token logits → `ScriptLine` fields + +**Edit** `src/senselab/audio/tasks/speech_to_text/huggingface.py` (Whisper path): + +```python +pipe(..., generate_kwargs={ + "language": ..., "num_beams": 1, + "return_dict_in_generate": True, + "output_scores": True, +}) +``` + +- From returned `scores` (per-step logits), compute per-token softmax entropy and `avg_logprob`; extract Whisper `no_speech_prob` where available. +- Attach onto each emitted `ScriptLine` (new optional fields — data-model §6): `avg_logprob`, `no_speech_prob`, `token_entropy`. +- Non-Whisper backends leave these `None` (graceful degradation, FR-017). + +**Edit** `src/senselab/utils/data_structures/script_line.py`: declare the three optional fields (default `None`) and map them in `from_dict`. This also **revives the existing dead `avg_logprob`/`no_speech_prob` reads** in `harvesters.py` (both presence and utterance). + +## B. Overlap grid handling — `utterance.py` + +- Utterance already receives its own `utterance_grid` (default 1.0 s / 0.5 s overlap) and already excludes boundary-straddling words (`asr_text_in_window(..., fully_contained=True)`). Contract confirms/keeps this and adds the token-entropy vote: + +```python +votes[m] = { + "text": ..., "phoneme_sequence": ..., "avg_logprob": ..., + "alignment_ctc_score": ..., + "token_entropy": mean_token_entropy_in_window(resolved, start, end), # NEW, None-safe +} +``` + +- `mean_token_entropy_in_window`: mean of per-token entropies whose timestamp midpoint ∈ `[start,end)`; `None` when the backend didn't supply token entropy. + +## C. Aggregator sub-signal — `aggregate.py::aggregate_utterance` + +Add one sub-signal to the existing fold (pairwise phoneme distance + Whisper `1−exp(avg_logprob)` + PPG): + +```python +# token entropy → uncertainty in [0,1] +te = mean over contributing models of normalized token entropy # normalize by log(vocab) or a fitted temperature +if te is not None: + sub_signals.append(te) +``` + +- Combined via the existing `--uncertainty-aggregator` (min/mean/max) — unchanged mechanism. +- Confidences from different backends mapped to a common calibrated `[0,1]` scale (FR-018) using the `CalibrationProfile.temperature` hook. + +## D. Scene-quality coupling — `compute.py` / `utterance.py` + +- After presence quality columns exist for the bucket's time span, compute a coupling multiplier: + +```python +coupling = 1.0 + w_q * quality_snr_at(span) + w_s * (src_machine + src_environment)_at(span) +utterance_uncertainty_coupled = min(1.0, utterance_uncertainty * coupling) +``` + +- `w_q`, `w_s` are parameters with documented defaults. The multiplier is written to the `scene_quality_coupling` column (recorded, not hidden — FR-019). The pre-coupling value remains available in `model_votes`/`raw_aggregated_uncertainty`. + +## E. Backward compatibility + +- When token entropy is `None` for all models (no Whisper / non-Whisper only), utterance falls back to today's sub-signals exactly (SC-008). +- `aggregated_uncertainty` column meaning is preserved; coupling is an additional column, and applied to the reported utterance uncertainty only per the documented rule. diff --git a/specs/20260722-175022-scene-quality-utterance/data-model.md b/specs/20260722-175022-scene-quality-utterance/data-model.md new file mode 100644 index 000000000..7a23b74d6 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/data-model.md @@ -0,0 +1,126 @@ +# Data Model: Scene-aware presence axis + improved utterance uncertainty + +**Feature**: `20260722-175022-scene-quality-utterance` | **Date**: 2026-07-22 + +Entities below extend existing structures in `src/senselab/audio/workflows/audio_analysis/types.py`, `io.py`, and `src/senselab/utils/data_structures/script_line.py`. **All new fields are additive with defaults** (slots dataclasses / Pydantic), so positional construction and existing readers keep working (D10). + +--- + +## 1. `UncertaintyRow` (extended) + +`types.py` — `@dataclass(slots=True)`. Existing fields unchanged: `start, end, axis, aggregated_uncertainty, contributing_models, model_votes, comparison_status, intensity_weight, raw_aggregated_uncertainty`. + +**New fields (all default `None`)** — populated only on the `presence` axis (and the utterance-only pair on the `utterance` axis): + +| Field | Type | Axis | Range / values | Meaning | +|---|---|---|---|---| +| `presence_confidence` | `float \| None` | presence | `[0,1]` | calibrated mean P(speech) across voters (= `presence_p_voice`) | +| `presence_uncertainty` | `float \| None` | presence | `[0,1]` | decisiveness uncertainty `1−│2p−1│` (= existing `aggregate_presence`) | +| `quality_snr` | `float \| None` | presence | `[0,1]` degradation | 0 = clean high-SNR; 1 = fully noise-dominated | +| `quality_clip` | `float \| None` | presence | `[0,1]` degradation | proportion/severity of clipping | +| `quality_reverb` | `float \| None` | presence | `[0,1]` degradation | from Brouhaha C50 (low C50 → high degradation) | +| `quality_bandwidth` | `float \| None` | presence | `[0,1]` degradation | 0 = full-band; 1 = strongly band-limited | +| `quality_uncertainty` | `float \| None` | presence | `[0,1]` | spread among independent SNR estimators in the analysis window | +| `src_speech` | `float \| None` | presence | `[0,1]` mass | share of scene mass = target/other speech | +| `src_people` | `float \| None` | presence | `[0,1]` mass | non-speech human sounds (laughter, cough, chatter) | +| `src_machine` | `float \| None` | presence | `[0,1]` mass | engine/HVAC/tools/vehicle | +| `src_environment` | `float \| None` | presence | `[0,1]` mass | wind/rain/water/birds/etc. | +| `src_dominant` | `str \| None` | presence | one of the 4 category names | argmax of the four masses | +| `token_entropy` | `float \| None` | utterance | `≥0` | mean per-token ASR entropy over the bucket | +| `scene_quality_coupling` | `float \| None` | utterance | `≥1.0` | multiplier applied to utterance uncertainty from scene quality/source (recorded, not hidden) | + +**Validation rules**: +- `src_speech + src_people + src_machine + src_environment ≈ 1.0` (±1e-6) when any source classifier ran; all `None` when both AST and YAMNet absent. +- `src_dominant == argmax(masses)`. +- All `quality_*` in `[0,1]` or `None` (null when Brouhaha/estimator unavailable, per FR-023) — never NaN masquerading as a value. +- `presence_confidence`/`presence_uncertainty` set on every presence row; `None` only if presence produced no vote at all. +- Detailed per-estimator raw values (Brouhaha dB SNR/C50, each DSP SNR, per-category class contributions) live in `model_votes` JSON, not as columns. + +--- + +## 2. Parquet columns (`io.py::write_axis_parquet`) + +Each new `UncertaintyRow` field above maps to one pyarrow column (`float64`, or `string` for `src_dominant`), appended to the existing column set. Column-projecting readers ignore unknown columns; the frozen contract already tolerates extras (`intensity_weight`, `raw_aggregated_uncertainty` precedent). Full schema in `contracts/presence-parquet-columns.md`. + +--- + +## 3. `SoundSourceCategoryMap` (new, versioned JSON) + +`workflows/audio_analysis/data/audioset_source_map.json` + +```json +{ + "version": "1", + "categories": ["speech", "people", "machine", "environment"], + "default": "environment", + "map": { "Speech": "speech", "Vehicle": "machine", "Wind": "environment", "Laughter": "people", "...": "..." } +} +``` + +- **Keys**: AudioSet display-name strings, covering the union of AST (`id2label`, 527) and YAMNet (class CSV, 521) vocab. +- **Invariant (SC-003)**: every class the classifiers can emit maps to exactly one category; unmapped → `default` with a logged warning. +- **Loaded once**, cached; consumed by `sound_sources.py`. + +--- + +## 4. `PerAxisGrids` (config, not a new type) + +Three `BucketGrid` instances threaded through `compute_uncertainty_axes`: + +| Axis | win_length | hop_length | Source | +|---|---|---|---| +| presence | 0.1 s | 0.02 s | new `presence_grid` param (default) | +| utterance | 1.0 s | 0.5 s | existing `utterance_grid` param | +| identity / shared | 0.5 s | 0.5 s | existing `grid` param | +| quality analysis (internal) | 0.5 s | 0.25 s | fixed constant in `quality.py`, recorded in provenance | + +Grid params recorded in each axis's `AxisResult.provenance` (FR-015). + +--- + +## 5. `CalibrationProfile` (new, versioned JSON) + +`workflows/audio_analysis/data/scene_quality_calibration.json` (or under `artifacts/` when freshly fit) + +```json +{ + "version": "1", + "snr": {"type": "linear_db_to_unit", "clean_db": 30.0, "floor_db": 0.0}, + "reverb_c50": {"type": "linear_db_to_unit", "clean_db": 30.0, "floor_db": -5.0}, + "bandwidth": {"nyquist_ref_hz": 8000.0, "rolloff_pct": 0.85}, + "temperature": {"presence": 1.0, "utterance": 1.0} +} +``` + +- Fitted by `scripts/calibrate_scene_quality.py` from synthetic mixtures (D9); consumed by `quality.py`/`calibration.py` to map dB→`[0,1]`. +- Absent profile → documented default normalization (uncalibrated but bounded). + +--- + +## 6. `ScriptLine` (extended — utils) + +`src/senselab/utils/data_structures/script_line.py` — add optional fields (Pydantic v2, default `None`): + +| Field | Type | Meaning | +|---|---|---| +| `avg_logprob` | `float \| None` | Whisper per-chunk avg logprob (revives dead signal) | +| `no_speech_prob` | `float \| None` | Whisper no-speech probability | +| `token_entropy` | `list[float] \| float \| None` | per-token softmax entropy (or mean) | + +- Populated only by the Whisper HF path (D7); `None` for all other backends → graceful degradation. +- `from_dict` extended to map these keys; existing construction unaffected. + +--- + +## Relationships + +``` +compute_uncertainty_axes + ├─ presence_grid / utterance_grid / grid ──> BucketGrid (×3) + ├─ frame_posteriors(seg-3.0, brouhaha) ─┐ + ├─ quality.py ── brouhaha(SNR,C50)+DSP+librosa+CalibrationProfile ─┐ + ├─ sound_sources.py ── AST/YAMNet full dist × SoundSourceCategoryMap ─┤ + ├─ presence.py/aggregate.py ── voters (+frame posteriors, −coarse) ──┼─> UncertaintyRow(presence, +cols) + └─ utterance.py/aggregate.py ── ScriptLine(token_entropy)+coupling ───┴─> UncertaintyRow(utterance, +cols) + └─> write_axis_parquet (+cols) +``` diff --git a/specs/20260722-175022-scene-quality-utterance/plan.md b/specs/20260722-175022-scene-quality-utterance/plan.md new file mode 100644 index 000000000..62539ef5f --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/plan.md @@ -0,0 +1,124 @@ +# Implementation Plan: Scene-aware presence axis + improved utterance uncertainty + +**Branch**: `20260722-175022-scene-quality-utterance` | **Date**: 2026-07-22 | **Spec**: [spec.md](./spec.md) +**Input**: Feature specification from `specs/20260722-175022-scene-quality-utterance/spec.md` + +## Summary + +Rework the `audio_analysis` workflow's **presence** axis into a scene-aware dimension and improve the **utterance** estimator, without breaking downstream consumers. The presence axis keeps its name and `aggregated_uncertainty` output; all new signals are **additive columns**. The work adds (1) per-bucket audio-quality degradation scores (SNR, clipping, reverb, bandwidth) built from existing senselab DSP plus a new `pyannote/brouhaha` model, (2) background sound-source category masses (speech/people/machine/environment) from a hand-authored AudioSet map over full AST/YAMNet distributions, (3) continuous frame-level speech posteriors (`pyannote/segmentation-3.0` + Brouhaha VAD) with per-axis grids and a confidence/uncertainty split, and (4) an overlapping word-scale utterance grid, calibrated confidence, token-level entropy (via newly-plumbed Whisper token logits + extended `ScriptLine`), and recorded scene-quality coupling. A synthetic clean+noise+RIR calibration harness fits the dB→[0,1] normalizations. See [research.md](./research.md) for the decision record (D1–D10). + +## Technical Context + +**Language/Version**: Python 3.11–3.12 (repo `requires-python = ">=3.11,<3.15"`), managed via uv +**Primary Dependencies**: pyannote-audio (existing — adds `segmentation-3.0` raw-scores + `brouhaha` via `Model`/`Inference`), transformers (AST + Whisper token logits), torchaudio + torchaudio-squim (existing), librosa (promote from transitive → explicit), numpy/scipy (calibration), pandas/pyarrow (existing parquet), jiwer (existing) +**Storage**: File-based — parquet under `//uncertainty/{presence,identity,utterance}.parquet`; checked-in category map JSON and calibration profile JSON under the package; validation artifacts under `artifacts/` +**Testing**: pytest via `uv run pytest`; unit tests under `src/tests/audio/workflows/audio_analysis/` and `src/tests/audio/tasks/{voice_activity_detection,speech_to_text}/` +**Target Platform**: macOS arm64 (unit CI) + Linux/EC2 GPU (model-heavy paths); library, no server +**Project Type**: Single-project Python library (senselab) — extends existing `audio` module +**Performance Goals**: Model inference is per-pass (once on whole 16 kHz mono audio), then bucketed; no per-bucket model calls. Quality DSP runs on a coarse 0.5 s analysis window. Adds ≤2 pyannote forward passes (segmentation-3.0, brouhaha) per pass beyond today's pipeline. +**Constraints**: Backward compatibility — `presence` axis name, parquet paths, LS track names, and `aggregated_uncertainty` values unchanged; new signals additive and null-safe when a model is unavailable (FR-023). Gated model loads must use `ensure_hf_model`/`local_files_only` (constitution VI). +**Scale/Scope**: One workflow module + one new VAD frame-posterior function + one new model loader + `ScriptLine`/Whisper token plumbing + a scripts calibration helper + tests. ~5 new files, ~8 edited files. + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-checked after Phase 1 design.* + +| Principle | Status | Notes | +|---|---|---| +| I. UV-Managed Python | ✅ | All commands `uv run …`; librosa added via `uv add`. | +| II. Encapsulated Testing | ✅ | Tests under `uv run pytest`; model-heavy paths guarded/mocked in unit tests, real-model checks on GPU CI. | +| III. Commit Early and Often | ✅ | One commit per phase/sub-task (spec already committed separately). | +| IV. CI Must Stay Green | ✅ | pre-commit + macOS unit tests per push; no watch loops. | +| V. Anti-Pattern Avoidance | ✅ | No `print` (use `logger`); mock isolation via `monkeypatch.setattr` (memory `feedback_mock_cache_pollution`); `(ImportError, RuntimeError)` guards for optional model imports; no circular imports in `dependencies.py`. | +| VI. No Unnecessary API Calls | ✅ | Brouhaha + segmentation-3.0 load via `ensure_hf_model` + `local_files_only`; models validated/pinned by revision (memory `project_model_revision_pinning`). | +| VII. Simplicity First | ⚠️ justified | New signals are always-on additive columns (no feature flag). Backward-compat of `aggregated_uncertainty`/paths is a real need (shared library, b2aiprep consumers), not a speculative shim — see Complexity Tracking. | +| VIII. No Hardcoded Parameters | ✅ | Grids, model ids/revisions, `--scene-top-k`, category-map path, calibration clip/SNR/RT60/output paths all parameters with defaults. | + +**Gate result**: PASS (one justified deviation recorded below). Re-check after Phase 1: **PASS — no new violations introduced by the design.** + +## Project Structure + +### Documentation (this feature) + +```text +specs/20260722-175022-scene-quality-utterance/ +├── plan.md # This file +├── spec.md # Feature spec (committed) +├── research.md # Phase 0 — decision record D1–D10 +├── data-model.md # Phase 1 — entities & column schema +├── quickstart.md # Phase 1 — how to run & validate +├── contracts/ # Phase 1 — interface contracts +│ ├── frame_posteriors.md +│ ├── quality.md +│ ├── sound_sources.md +│ ├── presence-parquet-columns.md +│ ├── utterance.md +│ └── cli.md +└── tasks.md # Phase 2 — created by /speckit.tasks (NOT here) +``` + +### Source Code (repository root) + +```text +src/senselab/audio/ +├── tasks/ +│ ├── voice_activity_detection/ +│ │ ├── frame_posteriors.py # NEW — segmentation-3.0 + brouhaha per-frame arrays +│ │ └── api.py # (unchanged public API; new fn exported) +│ ├── scene_quality/ # NEW package — brouhaha loader (VAD/SNR/C50) +│ │ ├── __init__.py +│ │ └── brouhaha.py +│ ├── classification/ +│ │ └── api.py # EDIT — honor full top_k already; call sites raise top_k +│ ├── speech_to_text/ +│ │ └── huggingface.py # EDIT — Whisper output_scores → token entropy/avg_logprob +│ └── features_extraction/ +│ └── (reuse torchaudio_squim.py, quality_control/metrics.py — no edits) +├── workflows/audio_analysis/ +│ ├── grid.py # EDIT — (unchanged struct; presence_grid threaded in compute) +│ ├── quality.py # NEW — per-bucket quality vector harvester +│ ├── sound_sources.py # NEW — AudioSet→category masses harvester +│ ├── data/audioset_source_map.json # NEW — versioned display_name→category map +│ ├── presence.py # EDIT — frame-posterior voters + coarse-voter demotion +│ ├── aggregate.py # EDIT — presence split surfacing; utterance token-entropy sub-signal +│ ├── utterance.py # EDIT — overlap grid handling; token-entropy vote; quality coupling +│ ├── compute.py # EDIT — presence_grid param; wire quality+sources; new columns +│ ├── types.py # EDIT — new UncertaintyRow fields (defaulted) +│ ├── io.py # EDIT — new pa.array columns +│ ├── plot.py / labelstudio.py / disagreements.py # EDIT — surface new sub-signals +│ └── calibration.py # NEW — load/apply fitted normalization profile +└── data_structures/ + └── (ScriptLine lives in senselab/utils/data_structures/script_line.py — EDIT: optional fields) + +scripts/ +├── analyze_audio.py # EDIT — --scene-top-k, --presence-grid, wire new outputs +└── calibrate_scene_quality.py # NEW — synthetic noise+RIR calibration harness + +src/tests/audio/ +├── workflows/audio_analysis/ +│ ├── quality_test.py # NEW +│ ├── sound_sources_test.py # NEW (incl. SC-003 full-coverage check) +│ ├── frame_posteriors_test.py # NEW +│ ├── grid_test.py # NEW (per-axis grid) +│ ├── compute_uncertainty_axes_test.py # EDIT — new columns, presence_grid +│ └── aggregate_test.py # EDIT — presence split, token-entropy sub-signal +└── tasks/speech_to_text/ + voice_activity_detection/ # EDIT/NEW — token logits, frame posteriors +``` + +**Structure Decision**: Single-project library extension. New signal-processing/model units are isolated files (`frame_posteriors.py`, `scene_quality/brouhaha.py`, `quality.py`, `sound_sources.py`, `calibration.py`) each with one responsibility and a narrow interface (contracts in `contracts/`); edits to existing workflow files are additive. The one cross-cutting edit outside the workflow is `ScriptLine` + Whisper token plumbing (D7), isolated to phase 4. + +## Implementation Phases (independently testable, map to spec user stories) + +- **Phase 1 — Scene quality (US1, P1)**: `scene_quality/brouhaha.py` loader; `quality.py` harvester (Brouhaha SNR/C50 + DSP clip + librosa bandwidth + estimator-spread uncertainty); new `quality_*` columns; librosa explicit dep. Tests: `quality_test.py`, synthetic noised/clipped/band-limited fixtures. Delivers FR-001…FR-006, SC-001/SC-002. +- **Phase 2 — Sound sources (US2, P1)**: raise `top_k`; `data/audioset_source_map.json`; `sound_sources.py` harvester; `src_*` columns. Tests: `sound_sources_test.py` incl. full-coverage assertion. Delivers FR-007…FR-010, SC-003. +- **Phase 3 — Temporal resolution (US3, P2)**: `frame_posteriors.py` (segmentation-3.0 + Brouhaha VAD); `presence_grid` param; frame-posterior voters + coarse-voter demotion; `presence_confidence`/`presence_uncertainty` columns. Tests: `frame_posteriors_test.py`, `grid_test.py`. Delivers FR-011…FR-015, SC-004. +- **Phase 4 — Utterance rework (US4, P2)**: overlap grid handling; Whisper `output_scores`→token entropy; `ScriptLine` optional fields; calibrated confidence; scene-quality coupling column. Tests: speech_to_text token test, `aggregate_test.py` updates. Delivers FR-016…FR-019, SC-005/SC-006. **Highest risk — see Complexity Tracking; splittable to a follow-up if it destabilizes core `ScriptLine`.** +- **Phase 5 — Calibration (US5, P3)**: `scripts/calibrate_scene_quality.py`; `calibration.py` profile load/apply; validation artifact. Delivers FR-020…FR-022, SC-007. +- **Cross-cutting**: FR-023 (null-safe degradation), FR-024 (LS/plot/disagreements surface new signals), FR-025 (Brouhaha via HF-token path) threaded through all phases; SC-008 (regression) guarded continuously. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| Backward-compat of `presence` axis name, parquet paths, LS tracks, and `aggregated_uncertainty` (constitution VII "no compat shims") | senselab is a shared library; b2aiprep and existing LS bundles read the `presence` parquet and track names. Renaming/altering them breaks external consumers with no in-repo signal. The maintainer explicitly chose "keep presence, additive columns." | Renaming to a `scene` axis (the clean-slate option) was considered and rejected in brainstorming precisely because of downstream breakage. Additive columns are the minimal change, not a speculative shim. | +| `ScriptLine` gains optional `avg_logprob`/`no_speech_prob`/`token_entropy` fields (touches a core data structure used across tasks) | FR-017 requires a token-level utterance signal; no backend exposes token scores today and `ScriptLine` drops extras, so the fields must be declared. This also revives the already-dead `avg_logprob` presence/utterance signal. | Carrying token scores in a side-channel dict keyed by span was considered; rejected because every existing consumer already expects data on `ScriptLine`, and a parallel structure would fork the transcript representation. Fields are optional/defaulted (null blast radius). | diff --git a/specs/20260722-175022-scene-quality-utterance/quickstart.md b/specs/20260722-175022-scene-quality-utterance/quickstart.md new file mode 100644 index 000000000..feb82f901 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/quickstart.md @@ -0,0 +1,104 @@ +# Quickstart: Scene-aware presence axis + improved utterance uncertainty + +**Feature**: `20260722-175022-scene-quality-utterance` + +## Prerequisites + +```bash +# Dev environment (adds librosa as an explicit dep during implementation) +uv sync --extra text --extra video --extra senselab-ai --extra nlp --group dev + +# HuggingFace token for gated pyannote models (segmentation-3.0, brouhaha) +export HF_TOKEN= +# Accept model conditions once at: +# https://hf.co/pyannote/segmentation-3.0 and https://hf.co/pyannote/brouhaha +``` + +## Run the full workflow (new signals on by default) + +```bash +uv run python scripts/analyze_audio.py --input tutorial_audio_files/.wav +``` + +Then inspect the presence parquet: + +```python +import pandas as pd +df = pd.read_parquet("/raw_16k/uncertainty/presence.parquet") +df[["start","end","presence_confidence","presence_uncertainty", + "quality_snr","quality_clip","quality_reverb","quality_bandwidth","quality_uncertainty", + "src_speech","src_people","src_machine","src_environment","src_dominant"]].head() +``` + +## Common variations + +```bash +# Keep the old 0.5 s presence resolution +uv run python scripts/analyze_audio.py --input clip.wav --presence-grid-win 0.5 --presence-grid-hop 0.5 + +# Skip the scene/quality model (quality columns null, faster) +uv run python scripts/analyze_audio.py --input clip.wav --no-scene-quality + +# Only persist top-50 scene classes instead of the full distribution +uv run python scripts/analyze_audio.py --input clip.wav --scene-top-k 50 +``` + +## Standalone API + +```python +from senselab.audio.workflows.audio_analysis import BucketGrid, compute_uncertainty_axes + +axis_results, incomparable = compute_uncertainty_axes( + passes=passes_summary, + grid=BucketGrid(), # identity/shared 0.5 s + presence_grid=BucketGrid(win_length=0.1, hop_length=0.02), + utterance_grid=BucketGrid(win_length=1.0, hop_length=0.5), + params={...}, + audio={"raw_16k": audio}, + scene_quality=True, + sound_sources=True, +) +``` + +## Calibration (optional, P3) + +```bash +uv run python scripts/calibrate_scene_quality.py \ + --clean tutorial_audio_files/.wav \ + --snr-sweep 30 20 10 5 0 --rt60-sweep 0.2 0.5 1.0 \ + --out src/senselab/audio/workflows/audio_analysis/data/scene_quality_calibration.json +# → writes the profile + a reported-vs-true validation plot under artifacts/ +``` + +## Validate the acceptance criteria + +```bash +# Full new test set +uv run pytest src/tests/audio/workflows/audio_analysis/ -v + +# Category-map coverage (SC-003) +uv run pytest src/tests/audio/workflows/audio_analysis/sound_sources_test.py -v + +# Regression: existing consumers unchanged (SC-008) +uv run pytest src/tests/audio/workflows/audio_analysis/{compute_uncertainty_axes_test,disagreements_test,labelstudio_test,plot_test}.py + +# Lint / types / spelling before pushing +uv run ruff format && uv run ruff check --fix && uv run mypy . && uv run codespell +``` + +## Phase-by-phase demo (each independently testable) + +| Phase | Command to see it work | +|---|---| +| 1 Quality | run workflow on a clip, confirm `quality_*` columns; noised region shows higher `quality_snr` | +| 2 Sources | run on a clip with background traffic, confirm `src_machine` elevated; coverage test green | +| 3 Temporal | `--presence-grid-win 0.1`, confirm finer buckets + `presence_confidence`/`presence_uncertainty` | +| 4 Utterance | run with a Whisper model, confirm `token_entropy` populated + `scene_quality_coupling` in utterance parquet | +| 5 Calibration | run the calibrate helper, confirm profile JSON + validation plot | + +## Notes / gotchas (from research) + +- Quality metrics use a fixed **0.5 s analysis window** broadcast onto the finer presence buckets — the reported quality resolution is 0.5 s even when presence buckets are 0.1 s (recorded in provenance). +- `--scene-top-k` must be large enough (default = full) for source masses to be meaningful; top-1 presence behavior is unchanged regardless. +- Token entropy only populates for the Whisper HF backend; other ASR backends leave it null (utterance falls back to today's signals). +- Gated pyannote models: first run downloads and caches; subsequent runs use `local_files_only` (no Hub calls). diff --git a/specs/20260722-175022-scene-quality-utterance/research.md b/specs/20260722-175022-scene-quality-utterance/research.md new file mode 100644 index 000000000..18f028054 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/research.md @@ -0,0 +1,167 @@ +# Research: Scene-aware presence axis + improved utterance uncertainty + +**Feature**: `20260722-175022-scene-quality-utterance` +**Date**: 2026-07-22 +**Inputs**: `spec.md`; handoff note `SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md`; three codebase research passes (quality plumbing, scene/source plumbing, presence/utterance internals). + +This document resolves every unknown needed to plan implementation. Each decision lists rationale and the alternatives rejected. + +--- + +## D1 — Frame-level speech posteriors (presence) + +**Decision**: Add a new function in `src/senselab/audio/tasks/voice_activity_detection/` that returns **continuous per-frame speech probability arrays + frame hop** (not `ScriptLine` segments), sourced from `pyannote/segmentation-3.0` raw scores via `pyannote.audio.Model.from_pretrained(...)` + `pyannote.audio.Inference(model)` (max over the speaker/powerset axis → P(speech)). Load through `ensure_hf_model` + token + `local_files_only` when cached. + +**Rationale**: The current VAD path uses the high-level `Pipeline` (`pyannote_vad.py:73`), which returns thresholded segments and discards the posterior — exactly the smoothing the handoff note flags. `segmentation-3.0` is not referenced anywhere in the repo today, so this is a genuinely new low-level extractor. `Inference` yields a `SlidingWindowFeature` at ~16.9 ms/frame that we aggregate within each reporting bucket. + +**Alternatives rejected**: (a) reuse `detect_human_voice_activity_in_audios` — returns segments only, smoothing lost. (b) TEN VAD / Silero — new dependency; the note ranks pyannote raw scores highest-leverage and zero-dep. Out of scope per spec. + +**Constitution**: VI — use `ensure_hf_model`/`hf_local_files_only` (`dependencies.py:329,356`) rather than the direct-token load the pipeline path uses, so cached runs skip the Hub. + +--- + +## D2 — Scene/quality model: `pyannote/brouhaha` + +**Decision**: Add `pyannote/brouhaha` as a new scene/quality model loaded via the same `Model.from_pretrained` + `Inference` pattern as D1 (it is a `pyannote-audio` multitask model). One forward pass yields per-frame **(VAD, SNR dB, C50 dB)**. Gated; reuse the existing `HF_TOKEN` flow; **no new pip dependency, no subprocess venv**. + +- `quality_reverb` ← Brouhaha C50 (dB → `[0,1]` degradation via calibrated normalization). +- `quality_snr` primary ← Brouhaha frame SNR (dB → `[0,1]`). +- Brouhaha VAD head → a second frame-posterior presence voter alongside D1. + +**Rationale**: Confirmed via HF (`pyannote/brouhaha`, library `pyannote-audio`, gated, trained on LibriSpeech+AudioSet+EchoThief+MIT-reverb, arXiv 2210.13248). It coexists with the existing pyannote-audio dependency; it is the only source in-repo for room-acoustics (C50) — no DSP reverb routine exists anywhere in senselab. + +**Alternatives rejected**: DSP C50 proxy (energy-decay/kurtosis) — lower fidelity, and the maintainer explicitly chose "add the model." Brouhaha via the `brouhaha-vad` pip package — redundant; the pyannote-audio load path is already available. + +**Constraint**: Brouhaha inference is per-pass (once on the whole 16 kHz mono audio), then bucketed — not per-bucket model calls. + +--- + +## D3 — Quality signal sourcing + the grid-resolution split + +**Decision**: Compute the four quality degradation scores on a **coarse internal analysis window (0.5 s / 0.25 s hop)** and broadcast each presence bucket's value from its containing analysis window. Sources: + +| Signal | Source | Notes | +|---|---|---| +| `quality_snr` | Brouhaha frame SNR (primary) cross-checked vs `spectral_gating_snr_metric`, `peak_snr_from_spectral_metric` (existing DSP) | agreement spread → `quality_uncertainty` | +| `quality_clip` | existing `proportion_clipped_metric` (cheap, slice-safe) | per bucket directly | +| `quality_reverb` | Brouhaha C50 | learned | +| `quality_bandwidth` | **new** minimal estimator: `librosa.feature.spectral_rolloff` (85% rolloff vs Nyquist) | see D8 | +| `quality_uncertainty` | normalized spread across the independent SNR estimators (Brouhaha vs the two DSP SNRs) in the window | high when estimators disagree | + +**Rationale**: Research confirmed the STFT-based SNR metrics need a ≥~128 ms slice and SQUIM needs mono/16 kHz and is only meaningful at ≥~0.5 s; they are unreliable at the 0.1 s presence grid. Decoupling *analysis resolution* (0.5 s) from *reporting grid* (presence 0.1 s) is precisely the handoff note's central point. Broadcasting a 0.5 s quality value across the finer presence buckets is honest (the value's true resolution is recorded in provenance). + +**`voice_signal_to_noise_power_ratio_metric` is NOT used per-bucket** — it runs VAD internally; if used at all it is computed once per pass. + +**SQUIM**: available (`extract_objective_quality_features_from_audios` → stoi/pesq/si_sdr) but **optional** — it is a heavier secondary cross-check for `quality_snr`/intelligibility, batched over analysis windows, gated on `torchaudio_available()`. Not required for the P1 slice; include only if cheap enough on the validation clip. + +**Alternatives rejected**: per-bucket quality at 0.1 s (unreliable numerics); `get_windowed_evaluation` as the driver (a single `None` window nukes the whole series — we slice + call metrics directly following the `embeddings.py:_slice_audio` + tail-anchored `_window_starts` pattern instead). + +--- + +## D4 — Sound-source categorization + +**Decision**: +1. **Raise `top_k`** for the AST and YAMNet windowed classification calls to the full label space (AST 527, YAMNet 521) via a new `--scene-top-k` CLI parameter (default: full), so per-window **full class-score vectors** persist. Top-1 consumers (`speech_presence_labels`, YAMNet veto) are unaffected — they index `labels[0]`. +2. Author a **checked-in, versioned JSON map** `{AudioSet display_name → category}` over the union of the AST + YAMNet vocab, category ∈ `{speech, people, machine, environment}`, with a documented default (`environment`) and logging on unmapped classes. +3. New harvester (mirrors `classification_top1_in_window`) sums per-window `scores` into the four category masses, normalizes to sum ~1, projects onto presence buckets by the existing center→nearest-window logic (`presence.py:335`). + +**Rationale**: Research found only top-5 is persisted today (`_classify_windowed` truncates), and no AudioSet ontology exists in the repo or deps — so the map is hand-authored from the flat display-name list. Raising `top_k` is the mandatory unlock; the HF pipeline already computes the full distribution, only the slice is truncated. + +**Alternatives rejected**: vendoring the full AudioSet `ontology.json` tree (heavier, and a flat keyword/label→category dict is sufficient for 4 coarse buckets); a dedicated model (PANNs/BEATs/HeAR) — out of scope (interface left open per FR-010). + +**Coverage test** (SC-003): an automated test loads the AST `id2label` (527) and the YAMNet class CSV (521) and asserts every class maps to exactly one category. YAMNet class names live in the model's bundled CSV read inside the subprocess — the test obtains the canonical list from the model asset or a vendored copy. + +--- + +## D5 — Per-axis grids + +**Decision**: Add a `presence_grid: BucketGrid | None` parameter to `compute_uncertainty_axes`, mirroring the existing `utterance_grid` plumbing (param at `compute.py:57`, resolve at `:371`, thread into the presence harvest at `:246`, record in provenance at `:301`). Defaults: presence `0.1 s / 0.02 s`, utterance `1.0 s / 0.5 s`, identity/shared `0.5 s`. Quality analysis window fixed at `0.5 s / 0.25 s` (D3), recorded in provenance. + +**Rationale**: Per-axis grids are already half-implemented (`utterance_grid` is a real, CLI-wired parameter); presence/identity currently share one grid. Adding `presence_grid` is a mechanical mirror. + +**Alternatives rejected**: a single global grid (can't satisfy the phone-scale presence vs word-scale utterance requirement); a full per-axis grid registry object (premature abstraction — YAGNI per constitution VII; two named optional params suffice). + +--- + +## D6 — Presence confidence/uncertainty split + coarse-voter demotion + +**Decision**: +- Surface **`presence_confidence`** (= `presence_p_voice`, already computed at `compute.py:259` but never written) and **`presence_uncertainty`** (= existing `aggregate_presence` output) as two new columns. `aggregated_uncertainty` stays = `aggregate_presence` for backward compatibility. +- Add the D1/D2 frame-posterior voters to the per-bucket vote dict. +- **Demote coarse voters** (AST/YAMNet whole-window tags, `no_speech_prob`, sentence-level ASR overlap) at fine grids: instead of each casting an equal binary `speaks` vote per bucket, they contribute a single down-weighted "context prior" term. Implemented as a per-voter weight in the mean, keyed on whether the voter's native resolution is coarser than the reporting grid (native grid already recovered via `_native_classification_grid`, `presence.py:124`). + +**Rationale**: The presence aggregator is already a weighted mean of per-voter p, not Shannon entropy (the docstring is stale). Confidence and uncertainty are both already computed; the rework is (a) exposing them and (b) adding continuous voters + demoting coarse ones so fine-grid agreement isn't inflated by voters that repeat one value across many buckets. + +**Alternatives rejected**: replacing the aggregator wholesale with a new entropy/dispersion formula — unnecessary; the mean-of-p already yields both quantities. Dropping coarse voters entirely — loses useful low-frequency context; demotion to a prior is the note's recommendation. + +--- + +## D7 — Token-level utterance uncertainty (highest risk) + +**Decision**: Plumb Whisper token logits through the HF speech-to-text path and derive per-token entropy: +1. In `speech_to_text/huggingface.py`, request `generate_kwargs={"return_dict_in_generate": True, "output_scores": True}` (Whisper) and compute per-token softmax entropy + `avg_logprob`/`no_speech_prob` from the returned scores. +2. Add **optional fields** to `ScriptLine` (`avg_logprob`, `no_speech_prob`, `token_entropy: list[float] | float | None`) — currently `extra="ignore"` drops them, so they must be declared. +3. New utterance sub-signal in `aggregate_utterance` folding mean token entropy per bucket; new vote field harvested in `utterance.py`. Degrades gracefully (None) for backends that don't expose token scores (Granite/Canary/Qwen subprocess/text-only). + +**Bonus finding**: this also **revives the currently-dead `avg_logprob` signal** — research showed `avg_logprob`/`no_speech_prob` are read via `seg_attr` but return `None` in production because `ScriptLine` never carries them (only test fixtures do). So plumbing token scores fixes an existing latent gap in both presence (Whisper `no_speech_prob` voter) and utterance (Whisper native confidence). + +**Rationale**: FR-017 explicitly requires a token-level sub-signal; no backend exposes it today, so it must be generated. Whisper's HF generate loop is the only backend that naturally surfaces token logits. + +**Risk / sequencing**: This is the one change touching a **core data structure** (`ScriptLine`) and a **non-workflow module** (`speech_to_text`), with broad blast radius (many tasks construct `ScriptLine`). It is sequenced last (phase 4), behind an additive, defaulted-`None` field change, with focused tests. If it destabilizes, it can be split to a follow-up PR without blocking phases 1–3 (flagged in Complexity Tracking). + +**Alternatives rejected**: n-best/beam disagreement (needs `num_beams>1`, changes decoding cost and output) — entropy from a single greedy pass is cheaper and sufficient; MC-dropout (separate future primitive per existing memory `project_mc_dropout_optional`). + +--- + +## D8 — Effective-bandwidth estimator + librosa dependency + +**Decision**: Implement `quality_bandwidth` from `librosa.feature.spectral_rolloff` (85% roll-off frequency) normalized against Nyquist → high degradation when energy is confined to a low band (telephone/codec). **Add `librosa` as an explicit `pyproject.toml` dependency** (it is currently only transitive via `audiomentations`, pinned `0.11.0`). + +**Rationale**: No rolloff/bandwidth/centroid routine exists in senselab. librosa is already importable and used in `quality_control/metrics.py`, so this adds no real install weight, but relying on a transitive dep is fragile — make it explicit. Rolloff-vs-Nyquist cleanly separates full-band from band-limited signals. + +**Alternatives rejected**: Praat `extract_spectral_moments` (`spectral_gravity`/`spectral_std_dev`) — returns whole-slice scalars, needs parselmouth, and centroid/spread is a weaker band-limit signal than rolloff. Kept as a documented fallback only. + +--- + +## D9 — Synthetic calibration harness + +**Decision**: A `scripts/` helper that, from a clean tutorial clip: +- mixes added noise (white/pink, generated with numpy — no new dep) at a target SNR sweep; +- convolves a **synthetic exponential-decay RIR** parameterized to a target RT60 (numpy; C50 derived analytically) — no `pyroomacoustics` dependency; +- runs the quality estimators on the mixtures, fits the dB→`[0,1]` normalization (and a temperature-scaling hook for confidences) against the known SNR/RT60, persists the fitted parameters to a checked-in calibration profile, and emits a reported-vs-true validation plot/table. + +**Rationale**: No labeled per-bucket SNR/reverb dataset exists (spec assumption). Synthetic mixtures give exact ground truth cheaply. Generating noise and a decaying-envelope RIR in numpy keeps it dependency-free (constitution VII). + +**Alternatives rejected**: `pyroomacoustics` image-source RIRs (new dep for marginal realism gain); real RIR corpora (EchoThief/MIT) — download weight and licensing; the synthetic decay RIR gives a known RT60/C50 which is what calibration needs. + +**Constitution VIII**: clip path, SNR sweep, RT60 targets, output path all CLI parameters with sensible defaults. + +--- + +## D10 — Additive output & backward compatibility + +**Decision**: Add new columns to `UncertaintyRow` (slots dataclass) **with defaults**, and matching `pa.array` columns in `write_axis_parquet`: +`presence_confidence`, `presence_uncertainty`, `quality_snr`, `quality_clip`, `quality_reverb`, `quality_bandwidth`, `quality_uncertainty`, `src_speech`, `src_people`, `src_machine`, `src_environment`, `src_dominant`, plus the utterance `token_entropy` and `scene_quality_coupling`. Per-vote detail (raw estimator values, category sub-scores) rides in the existing JSON `model_votes` — no schema churn there. `aggregated_uncertainty` semantics unchanged. + +**Rationale**: Precedent exists (`intensity_weight`/`raw_aggregated_uncertainty` were added after the original parquet contract). Column-projecting readers ignore unknown columns; the LS bundle / plot / disagreements consumers keep working (regression tests guard this — SC-008). + +**Constitution VII note**: new signals are **always-on additive columns** (null when a model is unavailable), **not** feature-flagged — simpler than a toggle and constitution-compliant. SC-008's "unchanged when disabled" is realized as "the existing `aggregated_uncertainty` computation is untouched by the new columns," verified by the existing regression suite. + +--- + +## Resolved unknowns summary + +| Unknown | Resolution | +|---|---| +| Frame posteriors source | `segmentation-3.0` + Brouhaha via `Model`+`Inference` (D1, D2) | +| Reverb estimator | Brouhaha C50 (D2) | +| SNR estimator | Brouhaha SNR + existing DSP SNR metrics (D3) | +| Quality at fine grid | coarse 0.5 s analysis window broadcast to presence buckets (D3) | +| Source class vectors | raise `top_k` to full; hand-authored category map (D4) | +| AudioSet ontology | none in repo → static display-name map (D4) | +| Per-axis grids | add `presence_grid` mirroring `utterance_grid` (D5) | +| Confidence/uncertainty split | surface `presence_p_voice` + `aggregate_presence` (D6) | +| Token entropy | plumb Whisper `output_scores` + extend `ScriptLine` (D7) | +| Bandwidth + librosa | librosa `spectral_rolloff`; make librosa explicit dep (D8) | +| Calibration data | synthetic noise + decay-RIR in numpy (D9) | +| Output additivity | new defaulted columns + JSON votes (D10) | diff --git a/specs/20260722-175022-scene-quality-utterance/spec.md b/specs/20260722-175022-scene-quality-utterance/spec.md new file mode 100644 index 000000000..4c98cd2fc --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/spec.md @@ -0,0 +1,195 @@ +# Feature Specification: Scene-aware presence axis + improved utterance uncertainty + +**Feature Branch**: `20260722-175022-scene-quality-utterance` +**Created**: 2026-07-22 +**Status**: Draft +**Input**: User description: "Rework the presence axis into a scene-aware dimension quantifying audio quality (SNR, clipping, reverb, bandwidth) and background sound sources (people/machine/environment), add per-axis temporal grids and frame-level posteriors, and improve the utterance uncertainty estimator" + +## Context + +The `audio_analysis` uncertainty workflow (`src/senselab/audio/workflows/audio_analysis`) emits three per-bucket uncertainty time series — `presence` (was a speaker present?), `identity` (was it the same speaker?), and `utterance` (what was said?). Today the presence axis collapses many coarse and fine voters into a single binary-vote Shannon entropy on a fixed 0.5 s grid, and background sound events are only used as a binary "speech / not-speech" cue. Users analyzing Bridge2AI-Voice recordings (speech plus cough/breathing tasks, often with people, machines, or environmental noise in the background) need the presence axis to also characterize **how good the signal is** and **what else is in the scene**, and they need the utterance axis to give a more trustworthy "what was said" uncertainty. A handoff note (`SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md`) established that the real bottleneck at short segments is temporal resolution and voter calibration, not tagger accuracy. + +This feature keeps the `presence` axis name and its existing `aggregated_uncertainty` output (no breaking change to downstream consumers) and adds scene-quality and sound-source columns alongside it, introduces per-axis temporal grids and frame-level speech posteriors, and reworks the utterance estimator. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Per-bucket audio quality on the presence axis (Priority: P1) + +A researcher runs the audio-analysis workflow on a clinical recording and wants to know, over time, how usable the signal is: where it is noisy, clipped, reverberant, or band-limited (e.g. telephone-quality). They read the `presence` parquet and see, per bucket, a set of `[0, 1]` degradation scores (0 = clean) for signal-to-noise ratio, clipping, reverberation, and effective bandwidth, plus a quality-uncertainty score reflecting disagreement among the independent quality estimators. + +**Why this priority**: Signal quality is the single most requested addition and it directly gates the trustworthiness of every other axis. It reuses estimators senselab already implements, so it delivers value with the least new machinery. + +**Independent Test**: Run the workflow on a tutorial clip and verify the `presence` parquet contains per-bucket `quality_snr`, `quality_clip`, `quality_reverb`, `quality_bandwidth` in `[0, 1]` plus a `quality_uncertainty` column, and that a clean clip scores low degradation while a synthetically noised/clipped clip scores higher. + +**Acceptance Scenarios**: + +1. **Given** a clean recording, **When** the workflow runs, **Then** every quality degradation score is near 0 across all buckets. +2. **Given** a recording with a segment of added broadband noise, **When** the workflow runs, **Then** `quality_snr` rises in the noised buckets and returns to baseline outside them. +3. **Given** a recording with digital clipping in one region, **When** the workflow runs, **Then** `quality_clip` is elevated only in the clipped buckets. +4. **Given** a telephone-band (≤4 kHz) recording, **When** the workflow runs, **Then** `quality_bandwidth` is elevated across the recording. +5. **Given** multiple independent SNR estimators disagree in a bucket, **When** the workflow runs, **Then** `quality_uncertainty` for that bucket is elevated. + +--- + +### User Story 2 - Background sound-source categorization (Priority: P1) + +A researcher wants to know when non-target sound sources are present and what kind they are — other **people** (laughter, cough, chatter), **machines** (engine, HVAC, tools, vehicle), or **environment** (wind, rain, birds, water) — separately from the target speech. They read the `presence` parquet and see, per bucket, the relative mass assigned to `speech`, `people`, `machine`, and `environment`, plus which category dominates. + +**Why this priority**: Knowing the interfering source type is essential for interpreting both quality and utterance uncertainty, and it is the second half of the "scene analysis" ask. It reuses the AST and YAMNet outputs the workflow already computes. + +**Independent Test**: Run the workflow on a clip containing speech over background traffic and verify the `presence` parquet has per-bucket `src_speech`, `src_people`, `src_machine`, `src_environment` (summing to ~1) and a `src_dominant` label, with `src_machine` elevated during the traffic. + +**Acceptance Scenarios**: + +1. **Given** a speech-only clip, **When** the workflow runs, **Then** `src_speech` dominates in speech buckets. +2. **Given** a clip with background machinery, **When** the workflow runs, **Then** `src_machine` is the elevated non-speech category in the affected buckets. +3. **Given** a clip with overlapping background talkers or laughter, **When** the workflow runs, **Then** `src_people` is elevated. +4. **Given** every AudioSet class the scene classifiers can emit, **When** the category map is applied, **Then** each class maps to exactly one of `{speech, people, machine, environment}` (complete, non-overlapping coverage). + +--- + +### User Story 3 - Fine temporal resolution for presence (Priority: P2) + +A researcher analyzing brief events (a cough onset, an inter-word breath) finds the fixed 0.5 s grid too coarse and the binary VAD segments too smoothed to localize them. They configure a finer presence grid and receive a smooth, per-bucket **speech-presence confidence** derived from continuous frame-level posteriors, with a separate **uncertainty** that rises where the model is genuinely ambiguous or the bucket straddles an onset. + +**Why this priority**: Temporal resolution is the handoff note's central technical unlock and underpins the quality and source signals at short spans, but it is more invasive than the additive quality/source columns, so it follows them. + +**Independent Test**: Run the workflow with the presence grid set to a 0.1 s window / 0.02 s hop and verify the presence series has the expected finer bucket count, a `presence_confidence` and a `presence_uncertainty` column, and that a hand-marked brief onset is localized to within one hop. + +**Acceptance Scenarios**: + +1. **Given** the default configuration, **When** the workflow runs, **Then** the presence axis reports on a 0.1 s window / 0.02 s hop grid while identity and utterance keep their own grids. +2. **Given** a bucket fully inside steady speech, **When** the workflow runs, **Then** `presence_confidence` is high and `presence_uncertainty` is low. +3. **Given** a bucket straddling a speech onset, **When** the workflow runs, **Then** `presence_uncertainty` is elevated relative to neighboring steady buckets. +4. **Given** coarse voters (whole-window scene tags, per-30 s no-speech probability, sentence-level transcripts), **When** the presence axis is computed on a fine grid, **Then** those voters do not cast identical per-bucket votes that inflate agreement; they contribute only as a slowly-varying context prior. + +--- + +### User Story 4 - Improved utterance uncertainty estimator (Priority: P2) + +A researcher relies on the `utterance` axis to flag where the transcription is untrustworthy. Today boundary effects on the fixed grid inflate word-error-rate disagreement, and the confidence scores are uncalibrated. They want an overlapping word-scale grid that stops penalizing words that straddle boundaries, calibrated confidence, an additional token-level uncertainty signal, and coupling so that utterance uncertainty rises where the scene quality is poor or a competing source masks the target. + +**Why this priority**: This is a distinct axis from the scene rework and depends on the quality signal from US1, so it is sequenced after it. It is high value because utterance uncertainty is what most reviewers act on. + +**Independent Test**: Run the workflow on a two-speaker clip with a noisy region and verify the `utterance` parquet uses an overlapping word-scale grid, exposes a token-level uncertainty sub-signal, and that utterance uncertainty in the noisy region is higher than the same transcript content in a clean region. + +**Acceptance Scenarios**: + +1. **Given** a word straddling a bucket boundary, **When** utterance uncertainty is computed, **Then** the straddling word does not inflate the disagreement for either adjacent bucket. +2. **Given** transcripts with per-token confidence available, **When** the workflow runs, **Then** a token-level uncertainty sub-signal is included alongside the existing pairwise and native-confidence sub-signals. +3. **Given** two buckets with identical transcript agreement but different scene quality, **When** utterance uncertainty is computed, **Then** the bucket with poorer quality / stronger competing source reports higher utterance uncertainty, and the coupling factor is recorded (not applied silently). +4. **Given** confidence scores from different ASR backends, **When** they are combined, **Then** they are mapped to a common calibrated `[0, 1]` scale. + +--- + +### User Story 5 - Synthetic calibration of quality and presence signals (Priority: P3) + +A developer needs the quality degradation scores and presence/utterance confidences to mean the same thing across recordings. Because no labeled dataset of per-bucket SNR/reverb is on hand, they run a helper that synthesizes calibration data by mixing a clean speech clip with known noise at controlled SNRs and convolving known room impulse responses at target reverberation times, then fits the normalization (and a temperature scaling) so the reported scores track the known ground truth. They review a validation artifact comparing reported vs. true SNR/reverberation. + +**Why this priority**: Calibration improves interpretability but the raw signals are usable (documented, uncalibrated) without it, so it is the lowest priority and can land last. + +**Independent Test**: Run the calibration helper on a tutorial clip, confirm it produces mixtures at known SNR/RT60, fits the degradation-score normalization, and emits a validation plot/table of reported vs. true values with error within a documented tolerance. + +**Acceptance Scenarios**: + +1. **Given** a clean clip and a target SNR sweep, **When** the helper runs, **Then** it produces mixtures at each target SNR and reports the estimator's response at each. +2. **Given** the fitted normalization, **When** applied to held-out mixtures, **Then** reported degradation increases monotonically with true degradation. +3. **Given** the calibration outputs, **When** a developer inspects them, **Then** a validation artifact shows reported-vs-true agreement and the fitted parameters are persisted for reuse. + +--- + +### Edge Cases + +- **Audio shorter than a bucket window**: the whole clip is treated as a single bucket for the affected axis. +- **A required model is unavailable** (e.g. the gated scene/quality model cannot be loaded without a token): the affected sub-signal columns are emitted as null/NaN with a recorded reason, and the axis still produces its other columns — the workflow does not abort. +- **Non-16 kHz or multi-channel input**: quality estimators that require mono 16 kHz receive a resampled/downmixed view; the original audio is not mutated. +- **Silent or all-zero buckets**: quality degradation is reported as undefined (null) rather than a misleading extreme, and presence confidence is low with low uncertainty. +- **Non-English transcripts**: the phoneme-based utterance cross-check remains gated off (as today); the new token-level and scene-coupling signals still apply where language-independent. +- **Scene classifier emits a class outside the category map**: it is assigned to a documented default category and the omission is logged so the map can be extended. +- **Overlapping speech (target plus background talker)**: presence remains high; the source axis reflects both `speech` and `people` mass rather than forcing a single winner. + +## Requirements *(mandatory)* + +### Functional Requirements + +#### Scene quality (US1) + +- **FR-001**: The presence axis MUST emit, per bucket, four audio-quality degradation scores in `[0, 1]` (0 = clean): signal-to-noise ratio, clipping, reverberation, and effective bandwidth. +- **FR-002**: The SNR and clipping degradation scores MUST be derived from senselab's existing audio-quality routines rather than newly implemented signal processing. +- **FR-003**: The reverberation degradation score MUST be derived from the added scene/quality model's room-acoustics (C50) output. +- **FR-004**: The effective-bandwidth degradation score MAY be a new minimal estimator; it MUST be documented and MUST distinguish full-band from band-limited (e.g. telephone-band) signals. +- **FR-005**: The presence axis MUST emit a per-bucket `quality_uncertainty` score reflecting disagreement among the independent quality estimators for that bucket. +- **FR-006**: All new quality columns MUST be additive to the existing `presence` parquet; the existing `aggregated_uncertainty` column and its meaning MUST be preserved. + +#### Sound sources (US2) + +- **FR-007**: The presence axis MUST emit, per bucket, the relative mass assigned to each of `speech`, `people`, `machine`, and `environment`, summing to approximately 1, plus a `src_dominant` label. +- **FR-008**: Sound-source categories MUST be derived from the scene classifiers (AST and YAMNet) the workflow already computes, via a checked-in, versioned map from the classifier label ontology to the four categories. +- **FR-009**: The ontology-to-category map MUST cover every class the classifiers can emit (complete, non-overlapping), with a documented default for unmapped classes and logging when the default is used. +- **FR-010**: The source categorization MUST be structured so an alternative dedicated sound-source model can be added later as a backend without changing the parquet schema. + +#### Temporal resolution & frame posteriors (US3) + +- **FR-011**: The workflow MUST support per-axis reporting grids (window/hop), with presence defaulting to a 0.1 s window / 0.02 s hop, utterance to a 1.0 s window / 0.5 s hop, and identity/others to 0.5 s. +- **FR-012**: The presence axis MUST derive speech presence from continuous frame-level posteriors (from the existing pyannote segmentation model's raw scores and the added scene/quality model's voice-activity output) aggregated within each reporting bucket, without routing through the segment-thresholding VAD pipeline. +- **FR-013**: The presence axis MUST report a per-bucket `presence_confidence` (calibrated mean speech probability) separately from a per-bucket `presence_uncertainty` (cross-voter disagreement plus within-bucket temporal instability). +- **FR-014**: On grids finer than the coarse voters' native resolution, coarse voters (whole-window scene tags, per-segment no-speech probability, sentence-level transcripts) MUST NOT be summed as equal per-bucket voters; they MUST contribute only as a slowly-varying context prior. +- **FR-015**: Grid parameters actually used MUST be recorded in each axis's output provenance. + +#### Utterance estimator (US4) + +- **FR-016**: The utterance axis MUST use an overlapping word-scale reporting grid and MUST exclude words that straddle a bucket boundary from that bucket's disagreement computation. +- **FR-017**: The utterance axis MUST include a token-level uncertainty sub-signal where the ASR backend exposes per-token confidence/entropy, degrading gracefully to the existing sub-signals when it does not. +- **FR-018**: ASR confidence signals from different backends MUST be mapped to a common calibrated `[0, 1]` scale. +- **FR-019**: Utterance uncertainty MUST increase where scene quality is poor or a competing non-speech source is present, and the coupling factor MUST be recorded in the output rather than applied invisibly. + +#### Calibration (US5) + +- **FR-020**: The system MUST provide a helper that synthesizes calibration data by mixing clean speech with known noise at controlled SNRs and convolving known room impulse responses at target reverberation times. +- **FR-021**: The helper MUST fit the degradation-score normalization (and a temperature-scaling hook for confidences) against the synthesized ground truth and persist the fitted parameters for reuse. +- **FR-022**: The helper MUST emit a validation artifact comparing reported vs. true SNR/reverberation. + +#### Cross-cutting + +- **FR-023**: When any model or estimator is unavailable, the affected columns MUST be emitted as null/NaN with a recorded reason and the rest of the axis MUST still be produced. +- **FR-024**: The Label Studio bundle, timeline plot, and disagreements index MUST surface the new presence sub-signals (quality and source) without breaking the existing tracks/rows. +- **FR-025**: The added scene/quality model MUST reuse the existing HuggingFace-token access flow and MUST NOT introduce a new isolated subprocess environment. + +### Key Entities + +- **Presence bucket record**: a time interval on the presence grid carrying the existing `aggregated_uncertainty`, the new `presence_confidence`/`presence_uncertainty`, the four `quality_*` degradation scores plus `quality_uncertainty`, and the four `src_*` masses plus `src_dominant`. +- **Sound-source category map**: a versioned mapping from the scene-classifier label ontology to `{speech, people, machine, environment}`, with a documented default. +- **Per-axis grid configuration**: window and hop per axis, recorded in provenance. +- **Calibration profile**: fitted normalization/temperature parameters derived from synthetic mixtures, persisted for reuse. +- **Utterance bucket record**: existing pairwise-WER, native-confidence, and PPG sub-signals plus the new token-level sub-signal and the recorded scene-quality coupling factor. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: On a clean tutorial clip, at least 95% of buckets report all four quality degradation scores below 0.1. +- **SC-002**: On a clip with a synthetically noised region, the mean `quality_snr` degradation inside the region is at least 0.3 higher than outside it. +- **SC-003**: The sound-source category map assigns every class the scene classifiers can emit to exactly one category (100% coverage, 0 overlaps), verifiable by an automated check. +- **SC-004**: A hand-marked brief onset (≤50 ms) is localized by the presence signal to within one reporting hop (≤20 ms at the default presence grid). +- **SC-005**: Boundary-straddling words contribute 0 to adjacent buckets' utterance disagreement, verifiable on a constructed fixture. +- **SC-006**: For matched transcript agreement, utterance uncertainty in a poor-quality region is measurably higher than in a clean region on the same clip. +- **SC-007**: On held-out synthetic mixtures, reported SNR/reverberation degradation increases monotonically with true degradation (rank correlation ≥ 0.9). +- **SC-008**: Existing downstream consumers of the `presence` parquet, Label Studio bundle, timeline plot, and disagreements index continue to work unchanged (existing regression tests pass and `aggregated_uncertainty` values are unchanged when the new features are disabled). + +## Assumptions + +- Users run the workflow via `scripts/analyze_audio.py` or the importable `compute_uncertainty_axes` API, on Bridge2AI-Voice-style recordings (speech plus cough/breathing, variable background). +- The added scene/quality model is `pyannote/brouhaha`, loaded through the existing pyannote-audio + HuggingFace-token path (gated, no new pip dependency, no subprocess venv). +- AST and YAMNet are already available in a default run and provide the AudioSet-style class scores that feed the source categories; when both are absent, source columns are null. +- The four sound-source categories (`speech`, `people`, `machine`, `environment`) are sufficient for v1; finer taxonomies and a dedicated health-acoustics model (e.g. HeAR) are out of scope but the interface leaves room for them. +- No labeled per-bucket SNR/reverberation dataset is available, so calibration is bootstrapped from synthetic clean+noise+RIR mixtures; a real labeled set can replace it later via the same fitting hook. +- The default effective-bandwidth estimator uses a spectral-rolloff-style measure; reuse of existing Praat spectral moments is an acceptable alternative documented in planning. +- Backward compatibility takes precedence: the `presence` axis keeps its name and `aggregated_uncertainty` output; all new signals are additive columns. + +## Out of Scope + +- Adding a dedicated sound-source or health-acoustics model (PANNs/BEATs/HeAR) as a backend — the interface is left open but no such model ships in this feature. +- Adding TEN VAD or any new frame-VAD dependency — frame posteriors come from models already in use plus the added pyannote model. +- Renaming the presence axis or restructuring the parquet paths / Label Studio track names. +- A user-facing labeling UI beyond the existing Label Studio bundle. +- Calibration against a real labeled clinical dataset (only synthetic bootstrapping is in scope). diff --git a/specs/20260722-175022-scene-quality-utterance/tasks.md b/specs/20260722-175022-scene-quality-utterance/tasks.md new file mode 100644 index 000000000..bce75a651 --- /dev/null +++ b/specs/20260722-175022-scene-quality-utterance/tasks.md @@ -0,0 +1,249 @@ +--- +description: "Task list for scene-aware presence axis + improved utterance uncertainty" +--- + +# Tasks: Scene-aware presence axis + improved utterance uncertainty + +**Input**: Design documents from `/specs/20260722-175022-scene-quality-utterance/` +**Prerequisites**: plan.md, spec.md, research.md (D1–D10), data-model.md, contracts/ + +**Tests**: INCLUDED — the spec's User Scenarios & Success Criteria are mandatory and constitution II requires encapsulated testing. Tests are written before implementation within each story. + +**Organization**: Grouped by user story (US1–US5) for independent implementation and testing. Task IDs are sequential in execution order. + +## Format: `[ID] [P?] [Story] Description with file path` + +- **[P]**: parallelizable (different files, no incomplete-task dependency) +- Absolute repo root: `/Users/satra/software/sensein/senselab` + +--- + +## Phase 1: Setup (Shared Infrastructure) + +- [X] T001 Promote `librosa` from transitive to an explicit dependency in `pyproject.toml` (align with pinned `librosa==0.11.0`) and refresh `uv.lock` via `uv add librosa` (D8). +- [X] T002 [P] Scaffold new modules with docstrings + `__future__` imports and no logic yet: `src/senselab/audio/tasks/scene_quality/__init__.py`, `src/senselab/audio/tasks/scene_quality/brouhaha.py`, `src/senselab/audio/tasks/voice_activity_detection/frame_posteriors.py`, `src/senselab/audio/workflows/audio_analysis/quality.py`, `src/senselab/audio/workflows/audio_analysis/sound_sources.py`, `src/senselab/audio/workflows/audio_analysis/calibration.py`, and `src/senselab/audio/workflows/audio_analysis/data/` (dir). + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**⚠️ CRITICAL**: The additive-output contract must exist before any story can emit its columns. + +- [X] T003 Add new defaulted fields to `UncertaintyRow` in `src/senselab/audio/workflows/audio_analysis/types.py` per data-model.md §1 (`presence_confidence`, `presence_uncertainty`, `quality_snr`, `quality_clip`, `quality_reverb`, `quality_bandwidth`, `quality_uncertainty`, `src_speech`, `src_people`, `src_machine`, `src_environment`, `src_dominant`, `token_entropy`, `scene_quality_coupling`) — all `= None`, after existing defaulted fields (slots dataclass). +- [X] T004 Add matching `pa.array` columns (float64, `src_dominant` string) to `write_axis_parquet` in `src/senselab/audio/workflows/audio_analysis/io.py` per contracts/presence-parquet-columns.md, and extend the `comparator_provenance` metadata with per-axis grid params + model id/revision placeholders. +- [X] T005 [P] Foundational regression guard: run `uv run pytest src/tests/audio/workflows/audio_analysis/{compute_uncertainty_axes_test,disagreements_test,labelstudio_test,plot_test}.py` and confirm they still pass with the new all-null columns present (SC-008 baseline). Fix any reader that breaks on unknown columns. + +**Checkpoint**: Rows/parquet can carry the new columns (all null) without breaking any existing consumer. + +--- + +## Phase 3: User Story 1 - Per-bucket audio quality (Priority: P1) 🎯 MVP + +**Goal**: `presence` parquet gains `quality_snr/clip/reverb/bandwidth` + `quality_uncertainty` per bucket. +**Independent Test**: run workflow on a clip; clean → all quality < 0.1; noised region → `quality_snr` up ≥0.3; clipped region → `quality_clip` up; telephone-band → `quality_bandwidth` up. + +### Tests for User Story 1 + +- [X] T006 [P] [US1] Write `src/tests/audio/workflows/audio_analysis/quality_test.py`: synthetic fixtures (clean tone/speech; +broadband noise at known SNR; hard-clipped; low-pass ≤4 kHz) asserting each `quality_*` responds correctly and stays in `[0,1]`; assert `quality_uncertainty` rises when SNR estimators are made to disagree (SC-001, SC-002). Mock Brouhaha frames so the test is model-free. + +### Implementation for User Story 1 + +- [X] T007 [P] [US1] Implement `scene_quality/brouhaha.py`: `extract_brouhaha_frames(...) -> list[BrouhahaFrames|None]` via `pyannote.audio.Model.from_pretrained` + `Inference`, loaded through `ensure_hf_model`/`local_files_only` + `get_huggingface_token()`, cached per `(repo,revision,device)`; returns per-frame `(vad, snr_db, c50_db)` + hop; `None` on load failure `(ImportError, RuntimeError)` (contracts/quality.md A; FR-025, FR-023). +- [X] T008 [US1] Implement `quality.py::harvest_quality_scores`: 0.5 s/0.25 s analysis window (constants), slice via the `embeddings.py:_slice_audio` + tail-anchored `_window_starts` pattern; `quality_snr` (Brouhaha SNR primary, cross-checked with `spectral_gating_snr_metric` + `peak_snr_from_spectral_metric`), `quality_clip` (`proportion_clipped_metric`), `quality_reverb` (Brouhaha C50), `quality_bandwidth` (`librosa.feature.spectral_rolloff` 85% vs Nyquist), `quality_uncertainty` (normalized SNR-estimator spread); broadcast analysis-window values onto presence buckets; raw values into `_raw` (contracts/quality.md B). +- [X] T009 [US1] Wire quality into `compute.py`: call `extract_brouhaha_frames` once per pass and `harvest_quality_scores` per presence bucket; populate the `quality_*` columns on presence `UncertaintyRow`s and stash `_raw` into `model_votes`. Guard behind `scene_quality: bool = True` kwarg. +- [X] T010 [US1] Null-safe degradation: when Brouhaha is `None`, emit `quality_reverb`/`quality_snr` null (keep `quality_clip`/`quality_bandwidth` from DSP), never abort (FR-023); add a targeted test case. +- [X] T011 [US1] Record quality analysis-window params + Brouhaha model id/revision in presence `AxisResult.provenance` (FR-015, memory `project_model_revision_pinning`). + +**Checkpoint**: US1 fully functional and independently testable (the MVP). + +--- + +## Phase 4: User Story 2 - Background sound-source categorization (Priority: P1) + +**Goal**: `presence` parquet gains `src_speech/people/machine/environment` + `src_dominant`. +**Independent Test**: clip with background traffic → `src_machine` elevated; speech-only → `src_speech` dominates; coverage test green. + +### Tests for User Story 2 + +- [X] T012 [P] [US2] Write `src/tests/audio/workflows/audio_analysis/sound_sources_test.py`: (a) `test_category_map_covers_all_classes` — load AST `id2label` (527) + vendored YAMNet class list (521), assert every class resolves to exactly one of the 4 categories with zero silent gaps (SC-003); (b) masses from a synthetic window sum to ~1 and `src_dominant == argmax`; (c) background-machine window → `src_machine` dominant. + +### Implementation for User Story 2 + +- [X] T013 [P] [US2] Author `workflows/audio_analysis/data/audioset_source_map.json` (schema data-model.md §3): union of AST 527 + YAMNet 521 display names → `{speech,people,machine,environment}`, `default="environment"`, `version="1"`. Include a small vendored YAMNet class-name list if the model asset isn't importable in-process. +- [X] T014 [US2] Raise persisted scene classes: add `--scene-top-k` (default full 527/521) in `scripts/analyze_audio.py` and pass `top_k` to the AST + YAMNet `classify_audios(...)` windowed calls so full per-window distributions persist; verify top-1 consumers (`speech_presence_labels`, YAMNet veto) unaffected (contracts/sound_sources.md B). +- [X] T015 [US2] Implement `sound_sources.py::harvest_source_categories`: load + cache the map; per presence bucket, sum AST+YAMNet window `scores` into category masses (mean of available classifiers), normalize to ~1, `src_dominant = argmax`; project via center→nearest-window; log unmapped classes once each (contracts/sound_sources.md C). +- [X] T016 [US2] Wire `src_*` columns into `compute.py` presence rows behind `sound_sources: bool = True`; stash per-category class contributions into `model_votes`. +- [X] T017 [US2] Null-safe when both AST and YAMNet absent → all `src_*` null (FR-023); add test case. + +**Checkpoint**: US1 + US2 both work independently on the presence parquet. + +--- + +## Phase 5: User Story 3 - Fine temporal resolution for presence (Priority: P2) + +**Goal**: per-axis grids + continuous frame posteriors with `presence_confidence`/`presence_uncertainty`. +**Independent Test**: `--presence-grid-win 0.1`; finer bucket count; hand-marked ≤50 ms onset localized within one hop (SC-004). + +### Tests for User Story 3 + +- [X] T018 [P] [US3] Write `src/tests/audio/workflows/audio_analysis/frame_posteriors_test.py`: synthetic `FramePosterior`, assert `mean_posterior_in_window` returns correct mean + within-window std over overlapping frames; empty overlap → nan handled. +- [X] T019 [P] [US3] Write `src/tests/audio/workflows/audio_analysis/grid_test.py`: `presence_grid` default (0.1/0.02) yields expected bucket count vs shared grid; per-axis grids coexist; provenance records each. + +### Implementation for User Story 3 + +- [X] T020 [P] [US3] Implement `frame_posteriors.py`: `extract_speech_frame_posteriors(...)` from `pyannote/segmentation-3.0` raw scores (`Model.from_pretrained`+`Inference`, max over speaker/powerset axis), `FramePosterior` dataclass, `mean_posterior_in_window` helper; `ensure_hf_model`/`local_files_only`; null-safe (contracts/frame_posteriors.md). +- [X] T021 [US3] Add `presence_grid: BucketGrid | None = None` to `compute_uncertainty_axes` (default 0.1/0.02), mirroring `utterance_grid` plumbing (resolve, thread into `harvest_presence_votes`, record in provenance). +- [X] T022 [US3] Add frame-posterior voters to `presence.py` per-bucket votes: seg-3.0 (T020) + Brouhaha VAD head (reuse T007 `BrouhahaFrames.vad`), each `mean_posterior_in_window` → `_vote_from_pvoice(mean, mean>=0.5)`; within-window std feeds uncertainty. +- [X] T023 [US3] Coarse-voter demotion in `presence.py`/`aggregate.py`: down-weight voters whose native resolution (via `_native_classification_grid`) is coarser than the reporting grid to a single context-prior term instead of an equal per-bucket vote (FR-014); add aggregate test. +- [X] T024 [US3] Surface `presence_confidence` (= `presence_p_voice`, already computed at `compute.py:259`) and `presence_uncertainty` (= `aggregate_presence`) as columns on presence rows; keep `aggregated_uncertainty` identical (FR-013, SC-008). +- [X] T025 [US3] Add `--presence-grid-win`/`--presence-grid-hop` to `scripts/analyze_audio.py`, wiring the new `presence_grid` (contracts/cli.md). + +**Checkpoint**: US1–US3 independently functional; presence reports on the fine grid with the split. + +--- + +## Phase 6: User Story 4 - Improved utterance uncertainty (Priority: P2) + +**Goal**: overlap grid + token-level entropy + calibrated confidence + scene-quality coupling. +**Independent Test**: run with a Whisper model on a noisy two-speaker clip → `token_entropy` populated, boundary-straddle words excluded (SC-005), utterance uncertainty higher in the noisy region (SC-006). +**⚠️ Highest risk** (touches core `ScriptLine` + Whisper path) — see plan Complexity Tracking; splittable to a follow-up PR without blocking US1–US3. + +### Tests for User Story 4 + +- [x] T026 [P] [US4] Write `src/tests/audio/tasks/speech_to_text/` token test: Whisper HF path with `output_scores` populates `ScriptLine.token_entropy`/`avg_logprob`/`no_speech_prob`; non-Whisper backend leaves them `None`. +- [x] T027 [P] [US4] Extend `src/tests/audio/workflows/audio_analysis/aggregate_test.py`: token-entropy sub-signal folds into `aggregate_utterance`; `scene_quality_coupling` multiplier raises reported uncertainty under poor quality; falls back exactly to today's signals when token entropy is `None`. + +### Implementation for User Story 4 + +- [x] T028 [US4] Add optional fields `avg_logprob`, `no_speech_prob`, `token_entropy` (default `None`) to `ScriptLine` in `src/senselab/utils/data_structures/script_line.py` and map them in `from_dict` (data-model.md §6). +- [x] T029 [US4] In `src/senselab/audio/tasks/speech_to_text/huggingface.py` (Whisper), request `generate_kwargs={"return_dict_in_generate": True, "output_scores": True}`, compute per-token softmax entropy + `avg_logprob` + `no_speech_prob`, attach onto emitted `ScriptLine`s (contracts/utterance.md A). This also revives the dead `avg_logprob` presence/utterance reads. + - **Deviation from contract A, as built**: passing `output_scores` through `pipe(generate_kwargs=...)` cannot work — transformers 5.5.4's `AutomaticSpeechRecognitionPipeline._forward` keeps only `sequences`/`token_timestamps` from the generate output and drops the scores before `postprocess`. Implemented instead as `speech_to_text/token_confidence.py`: a context manager that temporarily wraps `pipe.model.generate` to observe its output, leaving pipeline behavior untouched. It requests `output_logits` (raw) rather than `output_scores` (post-processor) because Whisper's logits processors suppress `<|nospeech|>` to `-inf`, which would pin `no_speech_prob` at 0 and distort the entropy. +- [x] T030 [US4] Add `mean_token_entropy_in_window` + a `token_entropy` vote field in `utterance.py::harvest_utterance_votes` (None-safe); confirm overlap grid + `fully_contained=True` boundary exclusion remain (SC-005). +- [x] T031 [US4] Fold the token-entropy sub-signal into `aggregate.py::aggregate_utterance` via the existing `--uncertainty-aggregator` mechanism (contracts/utterance.md C). +- [x] T032 [US4] Map ASR confidences (Whisper `avg_logprob`, CTC score) to a common calibrated `[0,1]` scale using the `CalibrationProfile.temperature` hook (FR-018). +- [x] T033 [US4] Compute + record `scene_quality_coupling` multiplier (from `quality_snr` + `src_machine`+`src_environment` over the bucket span, weights `w_q`/`w_s` params) in `compute.py`/`utterance.py`; apply to reported utterance uncertainty, keep pre-coupling value in `model_votes` (FR-019). +- [x] T034 [US4] Add `--utterance-scene-coupling-weights` CLI flag with documented defaults (contracts/cli.md; constitution VIII). + +**Checkpoint**: US1–US4 independently functional. + +--- + +## Phase 7: User Story 5 - Synthetic calibration (Priority: P3) + +**Goal**: fit dB→[0,1] normalization + temperature from synthetic mixtures; persist profile + validation artifact. +**Independent Test**: run the helper; reported degradation increases monotonically with true SNR/RT60 (rank corr ≥0.9, SC-007); profile JSON + plot emitted. + +### Tests for User Story 5 + +- [X] T035 [P] [US5] Write calibration test in `src/tests/audio/workflows/audio_analysis/` (or `src/tests/scripts/`): synthetic SNR/RT60 sweep → estimators → assert monotonic reported-vs-true and profile round-trips (load/apply) (SC-007). + +### Implementation for User Story 5 + +- [X] T036 [P] [US5] Implement `calibration.py`: `CalibrationProfile` load/apply (`linear_db_to_unit` etc., data-model.md §5) with a documented default when no profile is present. +- [X] T037 [US5] Implement `scripts/calibrate_scene_quality.py`: synthesize noise (white/pink, numpy) at an SNR sweep + convolve synthetic exponential-decay RIRs at an RT60 sweep (numpy, no new dep), run the quality estimators, fit normalization + temperature, persist to `data/scene_quality_calibration.json`; all inputs/outputs are CLI params with defaults (D9, constitution VIII). +- [X] T038 [US5] Emit a reported-vs-true validation plot/table under `artifacts/` (FR-022). +- [X] T039 [US5] Add `--calibration-profile` CLI flag (default bundled profile) and thread the profile into `harvest_quality_scores` + utterance calibration (contracts/cli.md). + +**Checkpoint**: all five stories independently functional. + +--- + + +> US5 + polish implemented 2026-07-24 (cowork session): `calibration.py` bridges the versioned +> §5 profile to the flat runtime dict both `quality.py` and `aggregate.py` consume; +> `scripts/calibrate_scene_quality.py` fits SNR anchors always and C50 anchors when Brouhaha is +> available, emits the FR-022 validation plot/table, and records provenance. NOTE: per-axis +> temperatures are CLI passthroughs (default 1.0) — a proper temperature fit needs labeled +> correctness (adaptive loop eval harness), tracked as the remaining US5 sub-item. +> +> **Fitted 2026-07-26 (first bundled profile).** Ran the fitter with Brouhaha available, so both +> the SNR and C50 anchors are measured rather than assumed; the SC-007 monotonicity gate passed. +> `data/scene_quality_calibration.json` now ships with the package and is the default (a run +> without `--calibration-profile` picks it up). +> +> The fit matters more than "one more artifact": Brouhaha's SNR estimate is strongly **compressed** +> (fitted slope 0.41), so a true 0–30 dB sweep only spans ≈4.7–16.6 dB of estimator output. The +> uncalibrated anchors (clean 25 dB / floor 5 dB) assumed true-dB scale and therefore scored a +> *pristine* 30 dB clip at **0.42 degradation**, with the whole scale squashed into [0.42, 1.0] — +> no dynamic range across the clean half, and FR-019 coupling silently inflating utterance +> uncertainty on clean speech (1 + 0.5·0.42 ≈ 1.21×). With the fitted anchors (clean 15.75 / +> floor 7.50, in estimator space) the same clip maps to 0.0. T045/T046 executed in the full +> environment. + +## Phase 8: Polish & Cross-Cutting Concerns + +- [X] T040 [P] FR-024: add `__presence__quality` and `__presence__sources` tracks to `labelstudio.py` (additive; existing `presence` track unchanged) + test in `labelstudio_test.py`. +- [X] T041 [P] FR-024: add optional quality + dominant-source rows to `plot.py` (existing 5 rows unchanged when new signals null) + test in `plot_test.py`. +- [X] T042 [P] FR-024: rank the new presence sub-signals in `disagreements.py` + test in `disagreements_test.py`. +- [X] T043 [P] Docs: update `workflows/audio_analysis/doc.md` and the CLAUDE.md workflow section describing the scene-aware presence columns + new CLI flags; cross-check against quickstart.md. +- [X] T044 Run quickstart.md end-to-end on a `tutorial_audio_files/` clip; confirm all new columns populate and acceptance scenarios hold. +- [x] T045 `uv run ruff format && uv run ruff check --fix && uv run mypy . && uv run codespell` (memory `feedback_ruff_format`). +- [x] T046 Full regression: `uv run pytest src/tests/audio/workflows/audio_analysis/` + confirm `aggregated_uncertainty` values unchanged vs baseline (SC-008). + +--- + +## Dependencies & Execution Order + +### Phase dependencies + +- **Setup (P1)** → **Foundational (P2)** → **User Stories (P3–P7)** → **Polish (P8)**. +- Foundational T003/T004 block every story's column emission; do them first. + +### Story dependencies + +- **US1 (P1)** — MVP; depends only on Foundational. Introduces `scene_quality/brouhaha.py` (T007), reused by US3. +- **US2 (P1)** — depends only on Foundational; independent of US1 (different harvester file). Shares `compute.py` as an integration point with US1 (sequence T009 and T016 to avoid edit conflicts). +- **US3 (P2)** — seg-3.0 posteriors (T020) are independent, but the Brouhaha VAD voter (T022) reuses US1's T007. Do US1 before US3's T022. +- **US4 (P2)** — scene-quality coupling (T033) reuses US1's quality columns. Do US1 before US4's T033. Otherwise independent; highest risk, can be deferred to a follow-up PR. +- **US5 (P3)** — calibrates US1's estimators; do after US1. + +### Within each story + +- Tests (T006, T012, T018/19, T026/27, T035) written first and expected to fail. +- Model/loader before harvester before `compute.py` wiring before provenance. +- `compute.py` is a shared integration file: tasks that edit it (T009, T016, T021, T024, T033) are **not** mutually `[P]` — sequence them. + +### Parallel opportunities + +- Setup T002 ∥ (after T001). +- Across stories after Foundational: US1 and US2 harvesters (`quality.py` vs `sound_sources.py`) develop in parallel; only their `compute.py` wiring serializes. +- Within a story, `[P]` tasks touch distinct files: e.g. T006 (test) ∥ T007 (loader); T012 (test) ∥ T013 (map JSON); T018 ∥ T019 ∥ T020. + +--- + +## Parallel Example: User Story 1 + +```bash +# Test + Brouhaha loader in parallel (distinct files): +Task: "Write quality_test.py with synthetic clean/noised/clipped/band-limited fixtures" +Task: "Implement scene_quality/brouhaha.py extract_brouhaha_frames" +# Then serialize the compute.py wiring: +Task: "Implement quality.py harvest_quality_scores" +Task: "Wire quality into compute.py presence rows" +``` + +--- + +## Implementation Strategy + +### MVP first (US1 only) + +1. Phase 1 Setup → 2. Phase 2 Foundational → 3. Phase 3 US1 → **STOP & VALIDATE** (`quality_*` columns on a real clip) → demo. This alone delivers the most-requested capability. + +### Incremental delivery + +US1 (quality) → US2 (sources) → US3 (temporal) → US4 (utterance) → US5 (calibration). Each merges as an independently testable increment; `aggregated_uncertainty` and existing consumers stay green throughout (SC-008). + +### Risk containment + +US4 is the only phase touching core `ScriptLine` + the Whisper path. If it destabilizes CI, split US4 to a follow-up PR — US1–US3 + US5 remain a complete, shippable increment. + +--- + +## Notes + +- `[P]` = different files, no incomplete dependency. `compute.py` edits never run `[P]` together. +- Commit after each task or logical group (constitution III); one focused commit per task ID where practical. +- No `print` — use `logger` (constitution anti-pattern #3). Mock isolation via `monkeypatch.setattr` (memory `feedback_mock_cache_pollution`). Optional-import guards use `(ImportError, RuntimeError)`. +- Gated pyannote models load via `ensure_hf_model` + `local_files_only` (constitution VI); pin revisions (memory `project_model_revision_pinning`). +- All commands via `uv run` (constitution I). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/architecture-review.md b/specs/20260723-225523-dynamic-uncertainty-workflow/architecture-review.md new file mode 100644 index 000000000..20593b7fe --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/architecture-review.md @@ -0,0 +1,142 @@ +# Architecture review — where should the PR's code live? + +> **Implementation status (2026-07-24)**: F1→T046, F2→T047, F3→T048, F4→T049 (minus the DSP-SNR +> move, blocked on splitting `quality_control/metrics.py`), F5→T050 are implemented in this PR and +> verified (15/15 unit + hermetic determinism e2e green; run15 reproduced all ground-truth metrics +> exactly). F6→T051 and F7→T052 remain sequenced follow-up PRs. Two policy knobs were added during +> implementation so degraded paths are explicit rather than raced: `u1_backend` and +> `audio_io_backend` (loader/backend recorded in provenance — fallbacks are never silent). + +**Date**: 2026-07-24 · **Scope**: everything this branch adds/touches (`votes.py`, `compute.py` +split, `adaptive/` subpackage, `scripts/analyze_audio.py` triage + stages, `scripts/adaptive_loop.py`, +tests) reviewed against senselab's documented architecture: `tasks//{__init__, api.py, +.py}` with model-type dispatch; Pydantic for cross-boundary `data_structures/`, dataclasses +for task/workflow internals (documented in `workflows/audio_analysis/types.py:1-4`); heavy imports +behind guarded/lazy patterns (`frame_posteriors.py:33-38` precedent). + +**Verdict in one paragraph.** The workflow-shaped code (loop, policy engine, belief store, regions, +convergence, triage decisions, LS export, plotting) is in the right place — +`workflows/audio_analysis/` is exactly senselab's home for composite, opinionated pipelines. What +does NOT belong where it currently sits is (a) **model capabilities re-implemented inside +`adaptive/backends.py` and `adaptive/audio_io.py`** that duplicate existing `tasks/` capabilities — +these exist only to survive the heavy-import problem, whose root cause is two eager parent +`__init__`s; (b) **generic utilities grown inside the workflow** (ScriptLine leaf-walk, word-level +WER, DSP SNR, transcript fusion) that have named homes in `tasks/` and `utils/`; and (c) the +**cache/provenance machinery still living in a 2,400-line script** (pre-existing debt this PR +deepened). None of these moves should happen inside this PR — the current state is verified and +byte-reproducible; each move below is a small, testable follow-up (tracked as T046–T052). + +## F1 — Import hygiene is the root cause; fix it before moving anything (T046) + +`import senselab.audio.workflows.audio_analysis.aggregate` pulls torch+speechbrain+transformers +because of exactly two eager `__init__`s: `audio/workflows/__init__.py:3` (imports +`explore_conversation`, which imports four model task stacks) and `audio_analysis/__init__.py:18-42` +(imports `compute` → `embeddings` → speechbrain). The repo already contains the fix pattern: +`adaptive/__init__.py` is PEP-562 lazy by design. **Recommendation**: lazify both parents the same +way (public API unchanged; `pdoc` handles `__getattr__` re-exports). This single change deletes the +`_ensure_light_importable` sys.modules stub in `scripts/adaptive_loop.py` — a hack that exists purely +to route around those two files — and is a precondition for F2/F3 (workflow code calling task APIs in +degraded environments). Also aligns with the standing import-time optimization effort +(`specs/20260501-154228-optimize-import-times`). + +## F2 — `adaptive/backends.py` re-implements task capabilities; dissolve it into `tasks/` (T047) + +| backends.py function | Existing home | What's missing there | +|---|---|---| +| `transcribe_crop` (HF whisper pipeline) | `tasks/speech_to_text` — `transcribe_audios([...], model=HFModel("openai/whisper-base"))` already works, model-agnostic (`huggingface.py:242-258`) | Nothing functional. U1 should build an `Audio` from the crop and call the task API; the pipeline-object cache (`_ASR_CACHE`) mirrors the backend's own `_pipelines` cache | +| `consensus_align` (torchaudio MMS_FA) | `tasks/forced_alignment` — has a **dead torchaudio slot**: `DEFAULT_ALIGN_MODELS_TORCH` (`constants.py:59-64`) and a `model_type == "torchaudio"` branch (`forced_alignment.py:173`) that nothing ever reaches | Wire MMS_FA as the real torchaudio backend of `align_transcriptions`; U3 then passes the consensus text through the standard API (and inherits its caching/`levels_to_keep` semantics) | +| `overlap_posteriors` (per-class powerset + pyannote-4.x multilabel handling + chunk stitching) | `tasks/voice_activity_detection/frame_posteriors.py` — `chunked_frame_inference` already returns the full `(frames, classes)` array and then discards classes 4-6 (`:86-88`); `FramePosterior` has no per-class field | This is FR-016's specified location. Add `per_class`/`overlap` to `FramePosterior` (or an `include_per_class=True` flag); backends.py's 3.x/4.x format handling and stitching should merge into `stitch_frames`/`_output_to_array`, not live in a workflow | +| `embed_windows` (fine-hop ECAPA) | `tasks/speaker_embeddings` + the workflow's own `embeddings.extract_per_window_embeddings` (same computation at coarse hop) | Parameterize the existing extractor (window/hop already args) and call it on crops; delete the duplicate | + +After T046+T047, `backends.py` reduces to thin guard wrappers (`(result, reason)` envelopes around +task APIs) or disappears entirely — guards can live in the intervention `guard` functions. + +## F3 — `adaptive/audio_io.py` duplicates preprocessing *and* creates a waveform-parity hazard (T048) + +`load_wav_16k_mono` (soundfile + `resample_poly`) and `crop` re-implement +`read_audios`/`resample_audios`/`extract_segments`. Beyond duplication there is a **correctness +nuance**: senselab's resampler and `resample_poly` produce different sample values, so an `Audio` +built from `audio_io` has a different `audio_signature` than the pipeline's own crop of the same +region — live U1/I4 results can never share cache entries with pipeline-produced crops +(contracts/region-reprocessing.md assumes they do). **Recommendation**: in full environments route +through `senselab.audio.tasks.preprocessing` (guaranteeing signature parity, requires F1); keep the +current numpy path only as the explicitly-labeled degraded-environment fallback. SepFormer +`_enhance` should likewise call `tasks/speech_enhancement.enhance_audios` rather than driving +speechbrain directly. + +## F4 — Generic utilities grown inside the workflow → named homes (T049) + +1. **ScriptLine leaf-walk**: ≥5 independent implementations (`fusion.iter_word_leaves`, two `_walk` + closures in `harvesters.py:170,701`, `forced_alignment.flatten_script_lines:550`, + `plotting.py:46`). Consolidate as `ScriptLine.iter_leaves()` (+ a dict-tolerant module helper for + serialized JSON) in `utils/data_structures/script_line.py`; migrate call sites opportunistically. +2. **WER**: two code paths — `tasks/speech_to_text_evaluation` (jiwer) vs the workflow's hand-rolled + `_levenshtein`/`_wer`/`_normalize_transcript_for_wer`. Keep `harvesters._levenshtein` (it works on + phoneme *sequences*, jiwer doesn't), but `adaptive/evaluate.py` should call `calculate_wer` with + the hand-rolled version as the no-jiwer fallback; move `_normalize_transcript_for_wer` to + `speech_to_text_evaluation/utils.py` where both consumers reach it. +3. **DSP SNR**: `triage.dsp_snr_series` (incl. the posterior-masked noise floor) belongs with the + three existing SNR estimators in `quality_control/metrics.py` — as `frame_snr_series_metric` / + posterior-masked variant. Blocker: `quality_control/metrics.py:11-14` imports VAD+diarization APIs + at top level (a heavy import for a "metrics" module — pre-existing debt worth splitting into pure- + DSP vs model-based halves while moving this in). +4. **LS ground-truth parsing**: `adaptive/evaluate.load_ls_ground_truth` (import side) should sit + next to the export side in `audio_analysis/labelstudio.py`. + +## F5 — Transcript fusion is a capability, not workflow plumbing (T050) + +`fuse_words` (ROVER-lite), `load_calibrator`, and `collect_word_streams` are model-independent, +ScriptLine-shaped, and pure — and there is no transcript-ensemble utility anywhere in `tasks/`. +**Recommendation**: promote to a new `tasks/speech_to_text_ensemble/` (naming parallels +`speech_to_text_evaluation/`), with the policy coupling removed at the boundary: accept an explicit +`weights: dict[model_id, float]` instead of the adaptive policy dict (the adaptive loop computes +weights via `policy.family_weights` and passes them in). The workflow keeps speaker/p_voice lookups +and `build_final_outputs` (those are workflow semantics). Defer until a second consumer exists if you +prefer rule-of-three — but the module is already dependency-free, so the move is cheap. + +## F6 — Cache/provenance machinery in the script (pre-existing, deepened by T012) (T051) + +`audio_signature`, `cache_key`, `align_cache_key`, `run_task_cached`, `run_alignment_cached`, +`_sync_cache_with_schema_version`, `wrapper_version_hash` are library-grade infrastructure inside +`scripts/analyze_audio.py`. T040 (in-process adaptive integration) cannot be built cleanly without +importing them, and the `_stage_*` functions (T012) belong in the workflow package once the cache +layer moves. **Recommendation**: `utils/tasks/cached_inference.py` (content-addressed outcome cache + +provenance envelope), then `workflows/audio_analysis/stages.py` for the six stage functions, leaving +the script a thin CLI. Note the `wrapper_version_hash` semantics change this implies: hash the +*stage/workflow modules* rather than the CLI file, so editing CLI plumbing stops invalidating the +model cache (make this an explicit, documented decision — it changes invalidation behavior). + +## F7 — Typed internals instead of dict soup (T052, opportunistic) + +House style for workflow internals is `@dataclass(slots=True)` (`types.py`, `BucketGrid`, +`WindowEmbedding` — Pydantic is deliberately reserved for cross-boundary types). The adaptive core +follows this for `Vote`/`PassHarvest` but passes `Region`, planner candidates/`InterventionRecord`, +election records, and the loop `ctx` (a 15-key god-dict shared with every intervention) as plain +dicts. data-model.md already specifies these entities. **Recommendation**: introduce +`adaptive/types.py` with `Region`, `PlannedIntervention`, `LoopContext` dataclasses; convert +incrementally (planner + regions first — they're the most-tested pure code). Also promote the +`RULES` list-of-dicts to a small `InterventionRule` dataclass or Protocol so rule authors get a typed +contract instead of a dict shape convention. + +## Correctly placed — leave alone + +`votes.py` (pure aggregate half) next to `aggregate.py`/`types.py`; `compute.harvest_pass` in the +workflow; `adaptive/{loop, policy, belief, regions, convergence, triage-decision, ls_final, plot, +evaluate-orchestration}.py`; `policy/default.yaml` as packaged data; the speckit docs; the e2e/env- +gated tests mirroring `src/tests/`; `scripts/make_degradation_suite.py` and `scripts/adaptive_loop.py` +as CLIs. `identity_repair.py` stays workflow-level (its calibration and consensus semantics are +uncertainty-workflow-specific), but its generic primitives (`_agglomerative_cosine`, +`detect_change_points`, `cross_source_disagreement`) are `utils/tasks/` candidates when a second +consumer appears — flagged, not moved. + +## Sequencing + +Nothing moves in this PR (the tree is verified, byte-reproducible, and every move above invalidates +that evidence). Follow-up order, each its own small PR with tests: + +1. **T046** lazy `__init__`s (unlocks everything; deletes the import stub hack). +2. **T047** dissolve `backends.py` into task APIs (incl. wiring the dead torchaudio aligner slot; + subsumes Phase-8 T041's posterior work via the FramePosterior extension). +3. **T048** `audio_io` → preprocessing routing (fixes the crop-signature parity hazard). +4. **T051** cache layer → `utils/tasks/cached_inference.py` + stages → workflow (enables T040). +5. **T049/T050/T052** utility consolidation, fusion promotion, typed internals — opportunistic. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/belief-store.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/belief-store.md new file mode 100644 index 000000000..c0bfc6df2 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/belief-store.md @@ -0,0 +1,53 @@ +# Contract: Belief store (votes + aggregated belief) + +Layout under `/`: + +```text +rounds/ +├── 0/ # triage +│ ├── belief/presence.parquet # presence-only (quality/scene/posterior voters) +│ ├── decisions.json # enhancement + no-speech gates (trigger values) +│ └── summary.json +├── 1/ # baseline +│ ├── belief/votes_{presence,identity,utterance}.parquet +│ ├── belief/{presence,identity,utterance}.parquet +│ ├── elections.json # per-region StreamElection +│ └── summary.json +├── / # k = 2..K intervention rounds +│ ├── belief/votes_*.parquet # ONLY rows added/updated this round (append-only design) +│ ├── belief/{presence,identity,utterance}.parquet # full re-aggregated state +│ ├── regions.json +│ └── summary.json +final/ ... # see contracts/final-outputs.md +``` + +## Rules + +1. **Append-only votes**: a round never rewrites earlier rounds' vote files. Shadowing (D5) is + expressed by setting `shadowed_by`/`status` on the *logical* vote via the current round's file; + readers reconstruct the live set as: latest status per vote id wins. +2. **Vote id**: `sha1(axis|bucket_start|source_model|stream|scope)` — deterministic; a re-fired + intervention on the same crop overwrites its own logical vote (idempotent). +3. **Shadowing**: `scope=region:*` from (model, stream) shadows `scope=file` from the same + (model, stream) in buckets covered by the region core. Never across models, never across streams. + `status=purged_hallucination` (from P3) excludes a vote from aggregation on **both** presence and + utterance axes; the row persists. +4. **Aggregation reads only `status=active` votes**, weighted by `policy.family_weights[family] × + payload.weight`. Aggregators are the existing pure functions (`aggregate.py`) plus the + epistemic/aleatoric decomposition (D7). +5. **Incremental re-aggregation**: after an intervention, only buckets intersecting its region core are + re-aggregated; all other rows carry forward by reference (same values, same `round`). +6. **Final-state mirroring**: the last round's aggregated rows are also written to the pre-existing + paths `/uncertainty/.parquet` and `uncertainty/raw_vs_enhanced/.parquet` with the + existing column set intact (FR-024); adaptive columns (`status`, `epistemic`, `aleatoric_floor`, + `elected_stream`, `irreducible_reason`, `round`) are additive. +7. **Provenance**: every parquet carries `schema.metadata` with `policy_hash`, `wrapper_hash`, + `senselab_version`, `round`, mirroring `write_axis_parquet` (`io.py:22`). + +## Invariants (tested) + +- No bucket loses evidence across rounds: `|active ∪ shadowed ∪ purged|` is non-decreasing. +- Re-running `aggregate_all` over the reconstructed live set reproduces every round's belief parquet + byte-identically (determinism, SC-004). +- For any (model, stream, bucket): at most one active vote. +- `history` in BeliefRow has exactly one entry per round that touched the bucket. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/cli.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/cli.md new file mode 100644 index 000000000..3909f9888 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/cli.md @@ -0,0 +1,50 @@ +# Contract: CLI (`scripts/analyze_audio.py`) + +All new flags default to values that preserve today's artifact set (FR-024). Existing flags unchanged. + +## New flags + +| Flag | Default | Meaning | +|---|---|---| +| `--max-rounds N` | `3` | Total rounds incl. triage+baseline. `1` = baseline only: no triage gating, no interventions, no `rounds/`≥2, `final/` still emitted from round-1 belief. | +| `--enhancement {auto,always,never}` | `always` | C1. `auto` gates the enhanced pass on triage quality; `always` = today's behavior; `never` ≡ existing `--no-enhancement` (kept as alias). | +| `--policy PATH` | packaged `default.yaml` | Policy file (FR-027). Overrides below win over the file. | +| `--budget-medium N` / `--budget-heavy N` | from policy | Per-run intervention budgets (FR-018). | +| `--max-region-rounds N` | from policy | Per-region intervention cap (FR-017). | +| `--region-top-n N` | from policy | Regions per round (FR-010). | +| `--reserve-asr-models M [M…]` | from policy | U2 escalation pool. | +| `--enable-overlap-separation` | off | v2 U4 rule (heavy). | +| `--no-adaptive-outputs` | off | Suppress `rounds/` + `final/` (debug/regression aid). | + +## Semantics & compatibility + +- **Golden-compat mode**: `--max-rounds 1 --enhancement always` → stage execution order, cache keys, + task JSONs, 9 uncertainty parquets, LS bundle, disagreements.json, summary.json pre-existing keys all + match today's outputs (SC-005). New outputs (`rounds/1/`, `final/`, additive summary keys, additive + parquet columns) appear unless `--no-adaptive-outputs`. +- **Cache-key stability**: triage and baseline reuse the existing task names and params in `cache_key` + (`scripts/analyze_audio.py:786`) — a triage AST/quality result is the same cache entry the baseline + would create; adaptive runs share cache with legacy runs of the same audio. +- **`--skip` interaction**: `--skip comparisons` disables the belief store and therefore all rounds ≥ 2 + and `final/` (a warning states this). Skipping a task removes its voters, nothing else. +- **Exit codes**: unchanged (0 success incl. `no_speech`; 2 usage/model-unavailable errors, e.g. the + existing brouhaha hard-fail stays). +- **Progress output**: each round prints one summary line (regions, fired/deferred, budget) — no + per-bucket spam. + +## Examples + +```bash +# Full adaptive run, default policy +uv run python scripts/analyze_audio.py audio.wav + +# Today's exact behavior + adaptive artifacts suppressed +uv run python scripts/analyze_audio.py audio.wav --max-rounds 1 --enhancement always --no-adaptive-outputs + +# Cost-capped unattended batch +uv run python scripts/analyze_audio.py audio.wav --enhancement auto --budget-medium 8 --budget-heavy 0 + +# Deep interrogation of one hard file +uv run python scripts/analyze_audio.py audio.wav --max-rounds 5 --region-top-n 16 \ + --reserve-asr-models openai/whisper-large-v3-turbo ibm-granite/granite-speech-3.3-8b +``` diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/final-outputs.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/final-outputs.md new file mode 100644 index 000000000..efdbdfd19 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/final-outputs.md @@ -0,0 +1,80 @@ +# Contract: Final outputs (`/final/` + additive attachments) + +## final/transcript.json + +```jsonc +{ + "calibrated": false, // true only when a calibration profile was applied (FR-021) + "policy_hash": "…", + "generated_from_round": 3, + "language": "en", + "words": [ + { + "text": "hello", "start": 1.24, "end": 1.51, + "speaker": "cluster_0", // unified cluster id; null when unattributable + "confidence": 0.93, + "sources": ["nyralabs/CrisperWhisper2.0_turbo", "Qwen/Qwen3-ASR-1.7B"], + "alternates": [], // [{text, share, models}] when winner margin < policy threshold + "flags": [] // overlap | low_presence | hallucination_purged_nearby | … + } + ], + "segments": [ /* utterance-level rollup: start, end, speaker, text, min_word_confidence */ ] +} +``` + +Invariants: words sorted by (start, end); `0 ≤ confidence ≤ 1`; every `speaker` exists in +final/diarization.json; every word derivable from active votes in the belief store (SC-008). + +## final/diarization.json (+ final/diarization.rttm) + +```jsonc +{ + "clusters": [{"cluster_id": "cluster_0", "member_labels": {"pyannote/…": ["SPEAKER_00"], "nvidia/…": ["spk0"]}, + "n_segments": 12, "total_speech_s": 43.1}], + "segments": [{"start": 0.8, "end": 4.2, "cluster_id": "cluster_0", + "boundary_confidence": {"start": 0.9, "end": 0.55}, // from I1 evidence; 0.5 = unrefined + "overlap": false}] +} +``` + +RTTM sidecar for interop. Cluster ids come from the existing unified clustering +(`clustering.py:202`); boundary confidences from I1 change-point evidence where it ran, else 0.5. + +## final/presence.parquet + +Fused presence at the presence grid: `start, end, p_voice, presence_confidence, status, +irreducible_reason?, elected_stream, overlap_posterior?`. This is the last round's presence belief — +identical values to `rounds//belief/presence.parquet`, republished for discoverability. + +## final/convergence.json + +Schema in data-model.md (ConvergenceReport). Additional invariants: `budget.spent` reconciles with +iterations.json (US5-3); `run_state=no_speech` runs contain only rounds[0]; every irreducible region's +`residual ≤ floor + epsilon` (D7). + +## final/iterations.json + +```jsonc +{ + "policy_hash": "…", + "entries": [ /* InterventionRecord, ordered by (round, plan order) — see data-model.md */ ] +} +``` + +Byte-identical across identical runs (SC-004). Includes `status: deferred_budget | blocked_guard` +entries — the full decision surface, not only fired actions. + +## Label Studio bundle (additive) + +- New tracks: `final__consensus_transcript` (TextArea, word-level regions with confidence in meta), + `final__presence` (Labels, converged/irreducible/open coloring), `final__diarization` (refined + boundaries). Existing per-model and per-axis tracks unchanged (FR-023, FR-024). +- disagreements.json gains a sibling `final/disagreements_resolved.json`: entries from the round-1 + index annotated with `resolution: converged | irreducible: | budget_exhausted` and the + intervention ids that touched them — the before/after story of the loop. + +## summary.json (additive keys) + +`adaptive: {run_state, rounds_executed, policy_hash, budget: {...}, uncertainty_mass_by_round: {...}}` +alongside the existing `global_uncertainty` block (whose semantics are unchanged; it is now computed +from the final belief state). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/interventions.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/interventions.md new file mode 100644 index 000000000..02aa739f9 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/interventions.md @@ -0,0 +1,124 @@ +# Contract: Intervention catalog (v1) + +Each rule declares: **trigger** (predicate over belief state), **guards**, **action**, **evidence +added** (votes), **cost class**, **expected_gain heuristic**. All parameters live in the policy file. +Rule ids are stable API (they appear in iterations.json and tests). + +## S1_stream_election — per-region stream choice (C5) + +- **Trigger**: region proposed on any axis; both streams have round-1 evidence; no election yet or + covered evidence changed. +- **Action**: score streams per policy weights over region buckets: presence confidence, quality + (1 − aggregated degradation), utterance agreement (1 − mean pairwise phoneme distance). Apply + enhancement-artifact guard: enhanced cannot win if raw-side PPG non-silent fraction < guard_min_raw_ppg + while enhanced-side claims speech, or squim STOI drops raw→enhanced. +- **Evidence**: `elected_stream` on region + covered BeliefRows; StreamElection record. +- **Cost**: light. **Gain**: prerequisite multiplier for U1/U2 (elected implicitly by those rules if + not yet run). + +## P2_fine_posteriors — fine-grid presence re-analysis + +- **Trigger**: presence region where coarse voters dominate (share of `coarse=true` active votes ≥ 0.5) + or `presence_uncertainty` driven by frame_instability. +- **Action**: re-run segmentation-3.0 + Brouhaha posteriors on the crop at fine hop (policy: + `fine_hop_s: 0.01`); per-class posteriors retained (FR-016). +- **Evidence**: replacement frame-posterior votes (scope region), overlap_posterior on covered rows. +- **Cost**: medium. **Gain**: `presence epistemic × mass`. + +## P3_hallucination_adjudication — ASR text where VAD says silence (C10) + +- **Trigger**: buckets with active ASR `speaks=true` votes while frame-posterior p_voice < 0.2. +- **Guards**: none (light; runs on existing evidence). +- **Action**: score indicators — whisper `no_speech_prob ≥ 0.5` with tokens (flag exists, + `presence.py:330`), `1 − exp(avg_logprob)` high, alignment CTC score low, PPG non-silent fraction + low, `src_machine + src_environment + music mass` high. Verdict `hallucination` if ≥ 2 independent + indicators (different families); verdict `missed_speech` if PPG active ∧ CTC high ∧ ≥ 2 ASR families + agree on text. +- **Evidence**: hallucination → affected utterance/presence votes `status=purged_hallucination` (C10); + missed_speech → add `adjudicator/missed_speech` presence vote `{speaks: true, weight: 0.5}` (C9). +- **Cost**: light. **Gain**: `n_affected_buckets × mean uncertainty`. + +## U1_region_reasr — targeted re-transcription + +- **Trigger**: utterance region, `epistemic ≥ theta_low` (disagreement-driven, not floor-driven). +- **Guards**: crop ≥ 1.0 s; elected stream audio available; region has ≥ 1 ASR family with active votes. +- **Action**: crop per contracts/region-reprocessing.md; run the round-1 ASR set on the elected stream + crop (cached, FR-014); auto-align text-only outputs (existing aligner path). +- **Evidence**: region-scoped utterance+presence votes per model (shadow same model+stream, D5). +- **Cost**: medium (per model forward on crop; counted once per rule firing). **Gain**: + `epistemic × uncertainty_mass × quality_gain_factor` (higher when elected stream ≠ round-1 dominant + stream — new information likely). + +## U2_reserve_escalation — add a dissenting model + +- **Trigger**: utterance region still open after a U1 firing (delta < ε or disagreement persists). +- **Guards**: reserve pool non-empty; heavy budget available; reserve model's family not already + majority in region. +- **Action**: run one reserve ASR model (policy `reserve_asr_models`, in order) on the elected-stream + crop; align if text-only. +- **Evidence**: new-model region-scoped votes (coexist). +- **Cost**: heavy. **Gain**: `epistemic × mass × (1 − family_overlap)`. + +## U3_consensus_realignment — authoritative word timestamps (C8) + +- **Trigger**: fusion round; contested or converged utterance regions with a consensus text. +- **Action**: force-align consensus text over the region (Qwen aligner default, MMS fallback — + script's existing backends). +- **Evidence**: `final` word timestamps; does not alter uncertainty rows. +- **Cost**: medium (batched once for all regions). **Gain**: fixed (fusion prerequisite). + +## I1_boundary_refinement — snap disputed diarization boundaries (C6) + +- **Trigger**: identity region containing ≥ 1 diar-model boundary where `__cross_diar_label_disagreement__` + or change_inconsistency is a top sub-signal. +- **Guards**: ≥ 1 embedding model available; crop ≥ 2 × embedding window. +- **Action**: re-embed crop at fine hop (policy `identity_fine_hop_s: 0.1`); compute calibrated + adjacent-cosine change-point trajectory (`calibrate_cosine_uncertainty`, `embeddings.py:666`); + emit change-point evidence. +- **Evidence**: `embedding_changepoint/` identity votes supporting the nearest model boundary + (or contradicting all, which raises epistemic honestly); refined boundary candidates for fusion. +- **Cost**: medium. **Gain**: `identity epistemic × mass`. + +## I4_overlap_detection — explain joint uncertainty (C7) + +- **Trigger**: co-located identity + utterance regions (time-IoU ≥ 0.5), or identity region with + rapid label alternation across models. +- **Guards**: per-class posteriors available (FR-016) — else fires P2 first (dependency declared). +- **Action**: compute overlap posterior = Σ multi-speaker powerset class probabilities over crop. +- **Evidence**: `overlap_posterior` on covered rows → `aleatoric_floor`; if floor explains residual, + regions close as `irreducible: overlapping_speech`. +- **Cost**: light (reuses P2 output) or medium (posteriors not yet fine-grained). +- **Gain**: `joint mass × co-location`. + +## U4_overlap_separation (v2, default off) + +- **Trigger**: irreducible `overlapping_speech` utterance region; `--enable-overlap-separation`. +- **Action**: SepFormer separation on crop → per-source embedding match to unified clusters → + per-source ASR (C11) → speaker-attributed region votes. +- **Cost**: heavy. + +## Implementation notes (contract aligned with code, 2026-07-24 — spec T045) + +- **U2 cost class is `medium`** (cache replay / one forward per crop), not heavy as originally + drafted; the "reserve family not already majority" guard was not implemented (the family-weight + aggregation already discounts intra-family agreement, making the guard redundant in practice). +- **`I2_recluster` is a catalog addition** beyond this contract (tasks.md T024a): change-point + + diar-boundary segmentation, p_voice-weighted pooling, cross-model co-association consensus; it + emits `embedding_recluster/consensus` votes and recomputes `__cross_diar_label_disagreement__`. +- **U3 runs as a fusion-stage step**, not a RULES entry — consistent with its trigger ("fusion + round") but not budget-ledgered; its guard is the SIGALRM timeout + backend availability + (`fusion.consensus_alignment` policy). +- **`max_region_rounds` is enforced via convergence marks** (per-bucket touch counts, FR-017) + rather than as a plan-time admission cap; the observable behavior (a region stops being + re-touched after N interventions without ε improvement) matches the contract's intent. +- **Backend pins**: `u1_backend` (`auto|senselab|pipeline`) and `audio_io_backend` + (`auto|senselab|dsp`) select the primary/fallback paths explicitly; the backend used is recorded + in `iterations.json` / `convergence.json → audio_backend` (never silent). + +## Shared guards + +- Region crops quantized to reporting grid before hashing (cache stability). +- A rule fires at most once per (region, round); `max_region_rounds` caps total firings per region + across rounds (FR-017). +- Any rule may be disabled in policy; disabled rules never appear in plans (but their triggers are + still evaluated for `next_actions` reporting). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/policy-engine.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/policy-engine.md new file mode 100644 index 000000000..b50b02e68 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/policy-engine.md @@ -0,0 +1,83 @@ +# Contract: Policy engine + +The engine is a pure function: + +```python +plan_round(belief: BeliefState, regions: list[Region], budget: BudgetLedger, + policy: Policy, round_idx: int) -> list[PlannedIntervention] +``` + +No I/O, no model loads, no randomness. Given equal inputs it returns an identical plan (FR-025). + +## Policy file (`adaptive/policy/default.yaml`) + +```yaml +version: 1 # bumped on any semantic change; sha256 of file → policy_hash +thresholds: + theta_speech: 0.5 # triage speech gate (FR-004) + theta_enh: 0.4 # quality degradation gate for --enhancement auto (FR-003) + theta_high: 0.66 # region seed (matches disagreements HIGH) + theta_low: 0.33 # converged (matches LS low bin) + epsilon: 0.05 # min per-intervention improvement (FR-017) +regions: + gap_merge_s: 0.5 + pad_s: 1.0 + top_n_per_round: 8 + max_region_rounds: 2 +budget: + medium_per_run: 24 + heavy_per_run: 4 + per_round_fraction: 0.5 # ≤ this fraction of remaining budget per round +election: + weights: {presence_conf: 0.4, quality: 0.3, utterance_agreement: 0.3} + guard_min_raw_ppg: 0.2 # enhancement-artifact guard (FR-015) +families: # FR-008 + whisper: ["openai/whisper-*", "nyralabs/CrisperWhisper*"] + nemo: ["nvidia/canary-*", "nvidia/stt_en_*"] + qwen: ["Qwen/Qwen3-ASR*"] + granite: ["ibm-granite/*"] +reserve_asr_models: ["openai/whisper-large-v3-turbo"] # U2 escalation pool +rules: # enable/disable + per-rule params + U1_region_reasr: {enabled: true} + U2_reserve_escalation: {enabled: true} + U4_overlap_separation: {enabled: false} # v2 + # ... every rule in contracts/interventions.md +``` + +## Scheduling + +1. Collect candidate (rule, region) pairs where the rule's trigger predicate holds and no guard blocks. +2. Score `priority = expected_gain(rule, region) / cost(rule)`; `expected_gain` is the rule's declared + heuristic over belief values (e.g. for U1: `epistemic × uncertainty_mass`), never a learned model. +3. Sort by (priority desc, axis priority utterance > identity > presence, region start asc, rule id) — + total order, no ties. +4. Greedily admit while class budgets and `per_round_fraction` allow; the rest are logged + `deferred_budget` and become `next_actions` candidates. +5. Execute admitted interventions **sequentially in sorted order** (deterministic vote-store state); + each runs in the failure envelope (D11). +6. Re-aggregate covered buckets; update region/bucket statuses (FR-017); end round. + +Loop termination (FR-019): `round_idx > max_rounds`, or zero fired interventions in a round, or all +regions closed. + +## Budget ledger + +`BudgetLedger` tracks caps/spent per class; every admit decrements atomically before execution; failed +interventions still count their class cost (they consumed the resources). Ledger state is serialized +into each `rounds//summary.json` and must reconcile with `final/convergence.json.budget` (US5-3). + +## Guards (evaluated before admission) + +- `region.status != open` → skip. +- `interventions_remaining == 0` → skip (region closes as irreducible/budget_exhausted per FR-017). +- Axis presence gate (C4): identity/utterance rules require region mean p_voice ≥ theta_speech unless + the rule is P3/C9 (which exist to adjudicate exactly that disagreement). +- Rule-specific guards declared in contracts/interventions.md (e.g. min crop length, stream + availability, model availability). + +## Determinism requirements + +- All floats compared with explicit rounding (1e-9) before ordering. +- `policy_hash = sha256(canonical yaml bytes)` recorded in every round summary, iterations.json, + convergence.json, and parquet metadata. +- The engine's unit tests replay recorded belief fixtures and assert byte-identical plans. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/region-reprocessing.md b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/region-reprocessing.md new file mode 100644 index 000000000..82ade79ce --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/contracts/region-reprocessing.md @@ -0,0 +1,50 @@ +# Contract: Region re-processing (crop → run → merge back) + +## Crop construction + +1. Input: Region with grid-quantized `core_start/core_end` (multiples of the axis reporting hop). +2. `crop_start = core_start − pad_s`, `crop_end = core_end + pad_s`, clipped to `[0, duration]`. +3. Trough snapping: within each pad, if presence p_voice has a local minimum < 0.2, move the crop edge + outward to that trough (cut in silence, not mid-word). Snapped edges re-quantized to the grid. +4. Extract with `extract_segments([(audio, [(crop_start, crop_end)])])` + (`senselab.audio.tasks.preprocessing`) from the **elected stream's** 16 kHz mono waveform. + +## Caching + +- The crop is an `Audio` whose `audio_signature` (`scripts/analyze_audio.py:741`) is content-derived — + existing `cache_key`/`align_cache_key` work unchanged (FR-014). No cache schema change. +- Grid quantization (step 1/3) guarantees identical regions across rounds/runs produce identical crop + signatures → cache hits. +- Provenance for crop-scoped outcomes adds: `{"crop": {"start": ..., "end": ..., "pad_s": ..., + "stream": ..., "region_id": ..., "parent_audio_signature": ...}}`. + +## Timestamp mapping + +- All model outputs on a crop are crop-local; merge-back adds `crop_start` to every start/end. +- Mapping is exact for word/segment timestamps; frame arrays record `frame_offset_s = crop_start`. + +## Merge-back (midpoint rule) + +- A word/segment/frame merges into the vote store iff its midpoint ∈ `[core_start, core_end)`. + Padded-context outputs are discarded (edge-effect quarantine, D2). +- Merged votes get `scope = region:` and shadow per contracts/belief-store.md §3. +- A model that returns nothing over the core (silence per that model) merges an explicit + `{speaks: false}` presence vote — absence of output is evidence, same convention as file scope. + +## Minimum-length and applicability rules + +| Runner | Min crop (post-pad) | Notes | +|---|---|---| +| ASR (any backend) | 1.0 s | Below this, skip U1/U2 (guard) | +| Forced alignment | same as its ASR text | Only for text-only outputs, unchanged from script | +| segmentation-3.0 / Brouhaha posteriors | 0.2 s | Chunked path already handles arbitrary length | +| Speaker embeddings (fine hop) | 2 × embedding window (default 2.0 s → 4.0 s) | I1 guard | +| YAMNet scene re-check | 0.96 s | Short-crop scene evidence | +| AST | never on crops | 10.24 s native window (`scripts/analyze_audio.py:61-67`) | +| Diarization | never on crops | D3 — global clustering context | + +## Failure + +A crop-run failure (model error, empty output where text was expected) follows D11: logged in +iterations.json, no vote changes, region's `interventions_remaining` still decremented (the attempt +consumed budget). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/data-model.md b/specs/20260723-225523-dynamic-uncertainty-workflow/data-model.md new file mode 100644 index 000000000..8589aadd2 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/data-model.md @@ -0,0 +1,128 @@ +# Data Model: Uncertainty-driven adaptive analysis workflow + +**Branch**: `20260723-225523-dynamic-uncertainty-workflow` | **Date**: 2026-07-23 + +All entities are Pydantic v2 models or frozen dataclasses (repo convention), serialized to parquet/JSON +under `/rounds//` and `/final/`. Existing entities (`BucketGrid` `grid.py:9`, +`UncertaintyRow` `types.py:24`, `AxisResult` `types.py:76`, `WindowEmbedding` `embeddings.py:45`) are +reused, not modified — new fields land on new entities. + +## Vote (row in VoteStore) + +The atom of evidence. One model's statement about one bucket on one axis. + +| Field | Type | Notes | +|---|---|---| +| `axis` | str | `presence` \| `identity` \| `utterance` | +| `bucket_start`, `bucket_end` | f64 | On that axis's reporting grid | +| `source_model` | str | Model id or synthetic source (`embedding_silhouette/`, `embedding_changepoint/`, `adjudicator/missed_speech`) | +| `family` | str | From policy family map (FR-008) | +| `stream` | str | `raw_16k` \| `enhanced_16k` | +| `scope` | str | `file` \| `region:` | +| `round` | int | Round that produced the vote | +| `payload` | struct | Axis-specific: presence `{speaks, native_confidence, weight, coarse, hallucinated}`; identity `{cluster_id, cosine_to_prev, ...}`; utterance `{text, phoneme_sequence, avg_logprob, alignment_ctc_score}` — same shapes the aggregators consume today | +| `shadowed_by` | str? | Vote id of the shadowing region-scope vote; shadowed votes are kept, not aggregated (D5) | +| `status` | str | `active` \| `shadowed` \| `purged_hallucination` | +| `provenance` | struct | cache_key, crop bounds (if region scope), intervention id, timestamp | + +Persisted: `rounds//belief/votes_.parquet` (append-only across rounds; each round file holds +that round's new/updated rows). + +## BeliefRow (aggregated state per bucket per axis) + +Extends the semantics of `UncertaintyRow` (all its columns retained for FR-024 compatibility) with: + +| Field | Type | Notes | +|---|---|---| +| `round` | int | Last round that changed this row | +| `status` | str | `open` \| `converged` \| `irreducible` \| `budget_exhausted` | +| `epistemic` | f64 [0,1] | Cross-source disagreement component (D7) | +| `aleatoric_floor` | f64 [0,1] | max(quality floor, overlap posterior) (C7, D7) | +| `overlap_posterior` | f64? | From per-class segmentation posteriors (FR-016) | +| `elected_stream` | str? | From S1 (FR-015); null before election | +| `irreducible_reason` | str? | `overlapping_speech` \| `snr_floor` \| `non_speech_vocalization` \| `single_model_coverage` \| ... | +| `history` | list | Uncertainty trajectory for monotonicity checks | + +Persisted: `rounds//belief/{presence,identity,utterance}.parquet`. The **final round's** rows are +also written to the existing paths (`/uncertainty/.parquet`) with the pre-existing column +set unchanged and new columns additive. + +## Region + +| Field | Type | Notes | +|---|---|---| +| `region_id` | str | `r__`, deterministic | +| `axis` | str | Proposing axis | +| `core_start`, `core_end` | f64 | Grid-quantized (D2) | +| `crop_start`, `crop_end` | f64 | Core ± pad, trough-snapped | +| `uncertainty_mass` | f64 | Σ (u − θ_low) · hop over seed+expanded buckets (ranking key, FR-010) | +| `elected_stream` | str | From S1 | +| `interventions_remaining` | int | Starts at `--max-region-rounds` | +| `status` | str | `open` \| `converged` \| `irreducible` \| `budget_exhausted` | + +Persisted: `rounds//regions.json`. + +## InterventionRecord (entry in iterations.json) + +| Field | Type | Notes | +|---|---|---| +| `intervention_id` | str | `__`, deterministic | +| `rule` | str | e.g. `U1_region_reasr` (contracts/interventions.md) | +| `region_id` | str? | Null for file-scoped rules (e.g. enhancement decision) | +| `trigger` | struct | Predicate inputs at decision time (reproducible from belief store, SC-008) | +| `action` | struct | Models run, stream, crop bounds, cache keys | +| `cost_class` | str | `light` \| `medium` \| `heavy` | +| `status` | str | `fired` \| `deferred_budget` \| `blocked_guard` \| `failed` | +| `error` | str? | repr(exc) when failed (D11) | +| `delta` | struct? | Per-axis mean/max uncertainty change over covered buckets, post re-aggregation | + +## StreamElection + +| Field | Type | Notes | +|---|---|---| +| `region_id` | str | | +| `scores` | map | Weighted per policy | +| `elected` | str | | +| `guard_fired` | bool | Enhancement-artifact guard (FR-015) forced raw | +| `guard_evidence` | struct? | Raw-side PPG/ASR values that contradicted enhanced-side speech | + +## RoundSummary (`rounds//summary.json`) + +`round`, `regions_proposed`, `interventions {fired, deferred, blocked, failed}`, `budget_spent by +class`, `uncertainty_mass {before, after} per axis`, `buckets {converged, irreducible} delta`. + +## ConvergenceReport (`final/convergence.json`) + +| Field | Type | Notes | +|---|---|---| +| `run_state` | str | `converged` \| `budget_exhausted` \| `max_rounds` \| `no_speech` | +| `rounds` | list | | +| `per_axis` | map | buckets total/converged/irreducible/open; residual uncertainty mass | +| `irreducible_regions` | list | The "explained" residual (D7) | +| `budget` | struct{caps, spent, by_class, by_rule} | Σ must equal iterations.json costs (SC US5-3) | +| `next_actions` | list | Top deferred interventions with priority — what more budget would buy (FR-018) | +| `policy_hash`, `wrapper_hash`, `senselab_version` | str | Determinism provenance (FR-025) | + +## FinalWord (row in `final/transcript.json.words[]`) + +| Field | Type | Notes | +|---|---|---| +| `text` | str | Winning candidate | +| `start`, `end` | f64 | From C8 consensus re-alignment when available, else weighted vote of member timestamps | +| `speaker` | str? | Unified cluster id at word midpoint; null in overlap unless v2 attribution ran | +| `confidence` | f64 [0,1] | Calibrated when profile exists; else raw weighted vote share (`calibrated` flag at document level) | +| `alternates` | list | Present when winner margin < policy threshold | +| `sources` | list | Contributing model ids (post family-weighting) | +| `flags` | list | `hallucination_purged_nearby`, `overlap`, `low_presence`, ... | + +Document level: `{calibrated: bool, policy_hash, generated_from_round, words: [...]}`. + +## Relationships + +```text +VoteStore ──aggregate_all──▶ BeliefRow ──propose──▶ Region ──match/rank──▶ InterventionRecord + ▲ │ + └────────────── new votes (shadow/coexist per D5) ◀── execute ─────────────┘ +BeliefRow(final) ──▶ fusion ──▶ FinalWord / final diarization / fused presence +RoundSummary* ──▶ ConvergenceReport +``` diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/plan.md b/specs/20260723-225523-dynamic-uncertainty-workflow/plan.md new file mode 100644 index 000000000..96bff40c7 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/plan.md @@ -0,0 +1,198 @@ +# Implementation Plan: Uncertainty-driven adaptive analysis workflow + +**Branch**: `20260723-225523-dynamic-uncertainty-workflow` | **Date**: 2026-07-23 | **Spec**: [spec.md](./spec.md) +**Input**: Feature specification from `specs/20260723-225523-dynamic-uncertainty-workflow/spec.md` + +## Summary + +Turn the single-shot analyze_audio pipeline into a bounded, deterministic, uncertainty-driven loop. +The three existing axes (presence / identity / utterance) become the control signal: a cheap triage +round gates enhancement and heavy tasks (C1/C2); a baseline round populates a persistent **belief +store** of provenance-tagged votes; intervention rounds propose high-uncertainty regions and execute a +declarative, budgeted **intervention catalog** on cropped audio (re-ASR on the elected stream, model +escalation, hallucination adjudication, boundary refinement, overlap detection); a fusion round emits a +consensus transcript, refined diarization, fused presence, and a complete decision audit trail. The +prerequisite refactor is a **harvest/aggregate split** of `compute_uncertainty_axes` so aggregation is +a pure, cheap function of the vote store. Decision record in [research.md](./research.md) (D1–D12). + +## Technical Context + +**Language/Version**: Python 3.11–3.12 (repo `requires-python = ">=3.11,<3.15"`), managed via uv +**Primary Dependencies**: no new runtime deps. Reuses pyannote-audio (segmentation-3.0 per-class +posteriors, Brouhaha), transformers, speechbrain (embeddings, enhancement), existing subprocess-venv ASR +backends, pandas/pyarrow, jiwer; PyYAML (already transitive via pre-commit/HF) for the policy file — +promote to explicit if not already a direct dep. +**Storage**: File-based — belief store + per-round artifacts under `/rounds//`; final +outputs under `/final/`; existing 9 uncertainty parquets, per-task JSONs, LS bundle unchanged. +Model-call caching stays in `artifacts/analyze_audio_cache/` (crop calls get distinct audio signatures). +**Testing**: pytest via `uv run pytest`; unit tests under `src/tests/audio/workflows/audio_analysis/` +(policy engine, merge semantics, region proposal, fusion — all pure-function testable without models); +GPU CI for end-to-end loop on the validation suite. +**Target Platform**: macOS arm64 (unit CI) + Linux/EC2 GPU (model-heavy paths); library + thin script. +**Project Type**: Single-project Python library (senselab) — extends `audio/workflows/audio_analysis`. +**Performance Goals**: Triage ≤ 15% of full-run cost; interventions bounded by explicit budget; clean +audio ≤ 60% of today's wall-clock (SC-002/003); re-aggregation after an intervention is O(covered +buckets), no model inference (FR-006). +**Constraints**: Backward compatibility (FR-024): `--max-rounds 1 --enhancement always` reproduces +today's artifact set; new outputs strictly additive. Determinism (FR-025): policy hash + seeded +clustering + stable ordering. No hardcoded parameters (FR-027). +**Scale/Scope**: One new subpackage (~8 files), a refactor of `compute.py` into harvest/aggregate, one +small extension to `frame_posteriors.py`, script restructuring into stage functions, policy YAML, +tests. ~10 new files, ~6 edited files. + +## Constitution Check + +| Principle | Status | Notes | +|---|---|---| +| I. UV-Managed Python | ✅ | All commands `uv run …`; any dep promotion via `uv add`. | +| II. Encapsulated Testing | ✅ | Policy engine / merge / region / fusion are pure functions — unit-tested without models; model paths mocked; loop e2e on GPU CI. | +| III. Commit Early and Often | ✅ | One commit per phase (A–E below). | +| IV. CI Must Stay Green | ✅ | Phase A lands behavior-neutral; regression goldens guard FR-024. | +| V. Anti-Pattern Avoidance | ✅ | `logger` not `print` in library code; `monkeypatch.setattr` mock isolation; optional-model `(ImportError, RuntimeError)` guards. | +| VI. No Unnecessary API Calls | ✅ | No new gated models; existing loaders (`ensure_hf_model`, pinned revisions) reused; interventions only load models already declared in policy. | +| VII. Simplicity First | ⚠️ justified | A loop + policy engine is intrinsically more machinery than a feed-forward script. Mitigation: policy-as-data, pure aggregation core, `--max-rounds 1` degenerates to today's pipeline. See Complexity Tracking. | +| VIII. No Hardcoded Parameters | ✅ | Every threshold/budget/weight in `policy/default.yaml` with CLI overrides (FR-027). | + +**Gate result**: PASS (one justified deviation recorded below). + +## Architecture + +### Control flow + +```text +main() + ├─ round 0 TRIAGE (raw_16k only) + │ quality DSP + brouhaha + seg3-posteriors(+per-class) + AST/YAMNet + openSMILE + │ → presence prior, quality map, source masses, overlap posterior + │ → decisions: no_speech? (FR-004 stop) enhancement auto? (FR-003) + ├─ round 1 BASELINE + │ run remaining stages on raw (+ enhanced if elected): diar, ASR, align, PPG + │ comparator HARVEST → VoteStore ; AGGREGATE → BeliefState v1 + │ stream election per coarse region (S1) + ├─ rounds 2..K INTERVENE (while budget ∧ open regions ∧ k ≤ max_rounds) + │ propose_regions(belief, axis) [regions.py] + │ plan = policy.match_and_rank(belief, regions, budget) [policy.py] + │ for each planned intervention: execute → new votes → VoteStore + │ AGGREGATE (covered buckets only) → BeliefState v(k) ; convergence marks + ├─ FUSION + │ consensus transcript (family/confidence-weighted word voting) + C8 re-alignment + │ final diarization (unified clusters + refined boundaries), fused presence + │ convergence report + iterations log + └─ existing outputs (9 parquets, LS bundle, disagreements.json, plot) + final/ + rounds/ +``` + +### New module layout + +```text +src/senselab/audio/workflows/audio_analysis/adaptive/ +├── __init__.py # run_adaptive_loop(...) public API +├── belief.py # VoteStore, BeliefState, merge/shadow semantics (FR-005..009) +├── regions.py # region proposal (FR-010), presence-trough snapping +├── policy.py # rule schema, ranking, budget, determinism (FR-011, FR-025) +├── interventions.py # catalog: P2,P3,C9,U1,U2,U3,I1,I4,S1 (FR-012) +├── reprocess.py # crop/pad/offset-map/merge-back via extract_segments (FR-013..014) +├── election.py # stream election + enhancement-artifact guard (FR-015) +├── convergence.py # thresholds, irreducibility, budget accounting (FR-017..019) +├── fusion.py # word-level voting, consensus re-alignment, final outputs (FR-021..023) +└── policy/default.yaml # all thresholds/budgets/family weights (FR-027) +``` + +Touched existing code: + +- `compute.py` — split into `harvest_all(...) -> VoteStore` and `aggregate_all(store, ...) -> + {(pass,axis): AxisResult}`; `compute_uncertainty_axes` becomes a thin wrapper (harvest → aggregate) + preserving its signature and stopping the in-place mutation of `passes` (`compute.py:236`) by + returning the synthetic-source injection as store entries instead. +- `frame_posteriors.py` — expose per-class powerset posteriors + `overlap_posterior` alongside the + existing collapsed P(speech) (`:78-88`); additive return field, existing callers unchanged (FR-016). +- `scripts/analyze_audio.py` — `run_pass` decomposed into per-stage functions (`stage_diarization`, + `stage_scene`, `stage_features`, `stage_asr`, `stage_alignment`, `stage_ppg`) with unchanged task + names/params (cache-key stability); `main` delegates to `run_adaptive_loop` with a legacy path for + `--max-rounds 1 --enhancement always`. +- `disagreements.py`, `labelstudio.py`, `plot.py` — additive: final consensus track, per-round deltas. + +### Key design points (full record in research.md) + +- **D1 deterministic policy engine** — declarative rules, priority = expected_gain/cost, stable + tiebreaks; no LLM in the control path. +- **D2 region cropping** — pad ± 1.0 s, snap to presence troughs, midpoint merge-back, min-length rules + (AST never on crops; YAMNet/posteriors for short crops); cache-native via crop audio signature. +- **D3 diarization stays whole-file** — global clustering context; local repair via embedding + change-points (I1) instead of re-diarizing crops. +- **D4 conditional whole-file enhancement + per-region stream election** — not per-region enhancement + in v1; enhancement-artifact guard before trusting the enhanced stream. +- **D5 vote merge** — shadow same (model, stream) at region scope; family weights against + double-counting. +- **D6 convergence** — θ reuse (0.33/0.66), ε = 0.05, max-region-rounds = 2, cost classes + light/medium/heavy. +- **D7 epistemic/aleatoric split** — epistemic = cross-source disagreement; aleatoric floor = + f(quality, overlap posterior); irreducibility requires the floor to explain the residual. +- **D8 harvest/aggregate split** — prerequisite for cheap iteration; VoteStore is the only interface + between rounds. +- **D9 fusion** — ROVER-style time-aligned word voting; C8 consensus re-alignment for authoritative + timestamps. +- **D10 policy-as-data** — YAML, hashed into provenance. +- **D11 failure envelope** — intervention failures logged, belief untouched, run continues (FR-026). +- **D12 v2 deferrals** — separation-based overlap re-ASR (U4/C11), LID re-run (U6), LLM advisor. + +## Project Structure + +### Documentation (this feature) + +```text +specs/20260723-225523-dynamic-uncertainty-workflow/ +├── spec.md +├── plan.md # This file +├── research.md # Decision record D1–D12 +├── data-model.md # VoteStore / BeliefRow / Region / InterventionRecord / ... +├── quickstart.md # How to run & validate +├── contracts/ +│ ├── belief-store.md +│ ├── policy-engine.md +│ ├── interventions.md +│ ├── region-reprocessing.md +│ ├── final-outputs.md +│ └── cli.md +└── tasks.md # created by /speckit.tasks (NOT here) +``` + +## Phasing (each phase lands green and independently valuable) + +- **Phase A — harvest/aggregate split + VoteStore (behavior-neutral)**: refactor `compute.py`; + `compute_uncertainty_axes` output bit-identical on goldens; VoteStore persisted under + `rounds/1/belief/`. Unlocks everything; zero user-visible change. +- **Phase B — triage + gating (C1, C2) + stage decomposition of run_pass**: `--enhancement auto`, + no-speech early exit, cache-key stability tests. Immediate compute wins (US1). +- **Phase C — loop core (US2 + parts of US3)**: regions.py, policy.py, budget/convergence, + interventions P2, P3, C9, U1, U2, S1; `rounds//` artifacts; iterations.json. +- **Phase D — identity/overlap repair + fusion (US3 + US4)**: I1, I4 (needs FR-016 posterior + extension), U3 consensus re-alignment, fusion.py, final/ outputs, LS consensus track, + convergence.json. +- **Phase E (v2, optional)**: U4 separation re-ASR behind `--enable-overlap-separation`, U6 LID, + advisory hook. + +Dependencies: A → B → C → D → E. B is shippable without C. Regression goldens (FR-024/SC-005) are +created in A and enforced from B onward. + +## Complexity Tracking + +| Deviation | Why needed | Simpler alternative rejected because | +|---|---|---| +| Policy engine + intervention catalog (VII) | The loop's value is *selective* compute; selection logic must be inspectable, testable, and overridable | Hardcoded if/else loop: untestable in isolation, violates FR-027, invites hidden coupling | +| Persistent per-round belief artifacts | Determinism audits (SC-004/008) and post-hoc debugging require replayable state | In-memory only: audit trail impossible, irreducibility unverifiable | +| Two execution paths in script (legacy vs loop) during transition | FR-024 golden-diff guarantee while phases land | Big-bang switch: violates "CI stays green", high regression risk | + +## Risks & Mitigations + +- **Oscillation / thrash** (intervention helps one model, hurts another): monotonicity guard (FR-017) + + per-region round cap; interventions never delete evidence, only add or shadow. +- **Crop-boundary artifacts**: context padding + midpoint merge-back (D2); tests with words straddling + crop edges. +- **Enhancement hallucination** (SepFormer inventing speech): election guard (FR-015) requires raw-side + phonetic corroboration before the enhanced stream wins. +- **Correlated evidence inflation**: family weights (FR-008) with tests that adding whisper-small to + whisper-large barely moves aggregated confidence. +- **Cache stampede on tiny region variations**: region boundaries quantized to the reporting grid before + cropping, so re-runs hit identical crop signatures. +- **US4/US5 of scene-quality-utterance unfinished**: P3 and calibration degrade gracefully (spec + Assumptions); loop lands independent of token-logit plumbing. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/prototype-results.md b/specs/20260723-225523-dynamic-uncertainty-workflow/prototype-results.md new file mode 100644 index 000000000..4d56fe3cc --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/prototype-results.md @@ -0,0 +1,236 @@ +# Prototype results — audio_48khz_mono_16bits vs human ground truth + +**Date**: 2026-07-24 · **Branch**: `20260722-175022-scene-quality-utterance` (uncommitted) · +**Code**: `src/senselab/audio/workflows/audio_analysis/adaptive/` + `scripts/adaptive_loop.py` · +**Input**: `artifacts/e2e_runs/audio_48khz_mono_16bits_20260724-001937` + `artifacts/analyze_audio_cache` · +**Ground truth**: Label Studio export `updated-label-a7a37522.json` (4.92 s, 4 speakers, 5 segments — +one deliberately left untranscribed by the annotator at 2.17–2.99 s) · +**Output**: `artifacts/adaptive_prototype/run2/` + +The prototype is artifact-driven: round 1 ingests a completed analyze_audio run into the belief store; +rounds 2–3 run the policy engine; reserve-ASR escalation replays `openai/whisper-large-v3-turbo` and +`ibm-granite/granite-speech-3.3-8b` from the content-addressable cache (no model loads anywhere). +Command: `python3 scripts/adaptive_loop.py --cache-dir artifacts/analyze_audio_cache +--ground-truth `. + +## Headline results + +**1. Harvest/aggregate split proven (D8/T007).** Re-aggregating every bucket purely from the vote +store reproduces all six stored parquets exactly: 540/540 buckets, max_abs_diff = 0.0, 0 mismatches. +Iteration is therefore free of GPU re-work by construction. + +**2. The loop attacks the right region and reduces uncertainty (US2).** The enhanced-stream utterance +axis peaked at 0.81/0.71 over buckets 2.0–3.5 s — covering exactly the span the human annotator could +not transcribe. Round 2: `U2_reserve_escalation` fired on that region (2 cached reserves, 18 +region-scoped votes, same-model shadowing) → mean uncertainty 0.609 → 0.519 (Δ −0.091). Round 3: U2 +re-fired, Δ 0.0 → ε-monotonicity guard + max-region-rounds closed both buckets as +`irreducible: no_reduction_under_available_interventions` (residual 0.72/0.83, quality floor 0.0 — +clean audio, so the residual is not noise; five model families simply disagree). The loop's verdict and +the annotator's blank textarea agree independently. + +**3. Fusion beats every individual model (US4).** Word-slot voting over 5 models (3 from the run + 2 +cache-replayed reserves), family-weighted (whisper-derived models share one family weight): + +| source | WER (transcribed GT regions) | normalized* | +|---|---|---| +| **fused consensus** | **0.059** | **0.000** | +| Qwen3-ASR-1.7B | 0.059 | 0.059 | +| granite-speech-3.3-8b | 0.059 | 0.059 | +| whisper-large-v3-turbo | 0.059 | 0.000 | +| canary-qwen-2.5b | 0.118 | 0.059 | +| CrisperWhisper2.0_turbo | 0.176 | 0.176 | + +\* annotator shorthand "u"→"you" equivalence. + +**4. Confidence localizes human uncertainty.** Mean fused word confidence inside the untranscribed GT +span: **0.36**, vs **0.49** in transcribed spans (with coverage-scaled abstention penalty; words there +carry `alternates` like And/I'm/Ranger). Presence: accuracy 0.913, recall 1.00 on the 0.1 s grid. +Identity uncertainty at GT speaker boundaries **0.92** vs **0.75** within segments — the axis points at +exactly the boundaries `I1_boundary_refinement` (deferred, `next_actions`) would refine; the known +diarization failure (pyannote merged all 4 speakers into one cluster) is surfaced, not hidden +(word-speaker accuracy 0.63 against the single cluster). + +**4b. Visual output.** `final/timeline.png` (`adaptive/plot.py`, best-effort like analyze_audio's +timeline): five aligned rows — ground truth (untranscribed span hatched), presence p_voice + +uncertainty band, identity uncertainty with GT boundaries dashed, utterance uncertainty round-1 vs +final with the proposed region, fired interventions (with Δ) and irreducible hatching, and the fused +words colored by confidence with alternates. + +**5. Loop bookkeeping honest and deterministic.** 13 interventions fired (S1 elections, C9 +missed-speech with recorded +Δ belief revisions, 2×U2), I1 blocked with `embedding_backend_unavailable` +into `next_actions`, P3 found nothing to purge (clean clip — correctly zero). Budget: 11 light, 2 +medium, 0 heavy. Two consecutive runs produce **byte-identical `iterations.json`** (SC-004 at prototype +scale). Unit tests: 7/7 (`adaptive_prototype_test.py`); ruff clean. + +## Full version (2026-07-24, second pass) — live interventions + identity repair + +New modules: `adaptive/identity_repair.py` (I1 change-points + I2 consensus re-cluster from per-window +embeddings), `adaptive/audio_io.py` (16 kHz load/crop, SepFormer enhanced-stream-on-demand), +`adaptive/backends.py` (live whisper re-ASR, live fine-hop embeddings, gated segmentation-3.0 overlap +posteriors — all guarded). Policy v2 adds `identity.*`, `u1_asr_models`, `enhancement_model`. Nothing +is tuned to the test clip; all parameters are policy defaults. + +Results on the same run dir (`artifacts/adaptive_prototype/run7`): + +| metric | baseline loop | full version | GT | +|---|---|---|---| +| predicted speakers | 1 cluster | **3 refined clusters** | 4 | +| word_speaker_accuracy | 0.63 | **0.80** | 1.0 | +| boundary precision / recall (±0.25 s) | — | **1.00 / 0.50** | — | +| fused WER (raw / normalized) | 0.059 / 0.000 | 0.059 / 0.000 (held) | 0 | +| confidence untranscribed vs transcribed | 0.36 < 0.49 | 0.36 < 0.49 (held) | — | + +Change-points landed at 0.875 s (GT 0.859) and 3.125 s (GT gap edge); the missed boundaries flank the +0.38 s "Kenny" segment — below what 0.25 s-hop stored embeddings resolve; the live fine-hop path +(`identity.fine_hop_s: 0.1`) exists for full environments. U1 fired live (whisper-base on the crop, +14 words), with the enhanced-stream request correctly falling back to raw +(`enhancement_backend_unavailable` recorded) — new dissenting evidence honestly raised measured +disagreement (+0.04) in the span the annotator also could not transcribe. Determinism re-verified +byte-identical with live U1 in the loop; 8/8 unit tests (incl. synthetic 3-speaker recovery test); +ruff clean. + +**Generalization check** (`artifacts/adaptive_prototype/gen1`): identical policy on +`english_conversation_higgs_audio_v2` (21.5 s, two-speaker argument) — parity 0/2306 buckets, 72 +interventions (44 S1, 9 I1, 9 I2, 5 U2, 4 U1 live, 1 C9; I4 guarded), refined clusters recover the +two dominant alternating speakers (R0 8.0 s / R3 8.9 s) with interjections split into minor clusters, +coherent speaker-attributed transcript, 9/24 medium budget. No crashes, no clip-specific behavior. + +## Third pass (2026-07-24) — granite dropped; fusion-stream rule fixed + +Investigating per-model behavior surfaced a stream-routing defect, not a model defect: on the +**enhanced** stream, SepFormer damaged the ASR — CrisperWhisper lost the whole 2.2–3.0 s span *and* the +final "you" (its 0.176 WER was an enhancement casualty; on raw it reads "…Kenny and Josh. We just +wanted to take a minute to thank you."), and Qwen3's only error was the same enhanced-dropped "you". +Region elections (presence 0.4 / quality 0.3 / agreement 0.3) still picked enhanced, and granite's +MMS-aligned words were silently patching the holes — dropping granite alone cost a word +(fused 0.118/0.059). + +**General fix** (`loop.py`): the fusion stream is now the stream with the lowest **final utterance +uncertainty mass** (transcripts come from the stream whose transcript evidence is most self-consistent; +elections remain for region re-processing). With granite removed from `reserve_asr_models` +(`run10`, raw-stream fusion, 4-model ensemble + live whisper-base): fused WER **0.059 / 0.000 +normalized**, word_speaker_accuracy **0.81**, confidence separation untranscribed-vs-transcribed +**0.42 / 0.95** (sharpest yet), determinism re-verified. CrisperWhisper on raw improves to 0.118; +whisper-base scores 0.294 alone but is family-weighted (whisper = {crisper, turbo, base}) and cannot +drag the consensus. Fused transcript: "This is Peter / This is Johnny / Kenny / *and and Joe I We* +(low-confidence, the untranscribed span) / we just wanted to take a minute to… thank you." + +## Fourth pass (2026-07-24) — triage implemented; I4 validated live; model refresh memo + +**Triage (US1, T010/T011)**: `adaptive/triage.py` (pure, unit-tested) + `--enhancement auto` wiring in +`scripts/analyze_audio.py`. Per SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md the speech gate uses continuous +segmentation-3.0 frame posteriors aggregated at ~100 ms (never segmentized VAD); SNR from Brouhaha with +a posterior-masked percentile-DSP fallback. Live validation (gated model, real token): +silent 3 s → `speech_present=False` (FR-004 stop); synthetic conversation → speech, **no enhancement** +(median 47.7 dB); clean+noise → **enhancement** (0.75 dB, low-SNR fraction 1.0); the archival test clip +→ enhancement under the DSP fallback (9.1 dB — borderline; Brouhaha is authoritative in production). +Conservative posture is intentional: triage gates *compute*, and the fusion-stream rule + +S1 guard already defend against a useless enhanced stream downstream. + +**I4 validated live** (`run13`): with HF token + pyannote 4.0.7, segmentation-3.0 scored the contested +region (288 frames) with **mean overlap 0.0** — the untranscribed span is a single unintelligible +speaker, not overlapped speech, so `no_reduction_under_available_interventions` is the *correct* +irreducibility reason. Stream fallback recorded (overlap is scene-invariant; raw waveform used when the +enhanced stream can't be materialized). `next_actions` is now empty: **every rule in the v1 catalog has +executed live at least once.** pyannote-4.x output format (per-chunk multilabel speaker activations) +handled alongside the 3.x powerset format in `backends.overlap_posteriors`. + +**Model landscape refresh**: see [research-models-2026.md](./research-models-2026.md) — three +HF-verified surveys (scene/events, diarization/VAD/embeddings, enhancement/separation/TSE) with a +phased adoption plan. Headlines: CED replaces AST (527-class, variable-length — removes the 10.24 s +crop prohibition); Streaming Sortformer v2 + DiariZen replace Sortformer v1 for speaker counting; +MarbleNet v2 / TEN-VAD give the triage gate an ungated frame-posterior backend; MossFormerGAN_SE_16K / +DeepFilterNet3 replace SepFormer with a do-no-harm SI-SDR gate; ReDimNet/ERes2NetV2 target the +unresolved 0.38 s speaker. Independent literature (arXiv 2512.17562, URGENT 2025) confirms the +raw-authoritative / gated-enhancement design. + +## Fifth pass (2026-07-24) — remaining tasks implemented; tasks.md fully checked + +**T008/T009 — harvest/aggregate split in the comparator itself.** New pure module `votes.py` +(`PassHarvest` + `aggregate_pass` + `compute_pass_deltas`, aggregation math moved verbatim; +unit-tested); `compute.py` refactored to `harvest_pass` (all model-touching work, no caller +mutation) + a signature-compatible `compute_uncertainty_axes` wrapper (`mutate_passes=True` default +preserves the timeline-plot contract; `False` gives the clean API). `VoteStore.from_harvests` +closes the in-process loop-integration path. Sandbox verification: compile, lint, 3/3 votes tests; +run the full `src/tests/audio/workflows/audio_analysis/` suite on a full env before merging. + +**T012 — run_pass stage decomposition**: six `_stage_*` functions, pure code motion, cache keys +unchanged. Note (applies to every script edit incl. triage): `wrapper_version_hash` is a whole-file +sha256, so the first post-merge analyze_audio run rebuilds the model cache once by design. + +**T031/T032/T033 — fusion follow-ups** (validated on `run14`): U3 consensus re-alignment via +torchaudio MMS_FA with SIGALRM timeout guard (`transcript.json.timestamps` records source/fallback +reason; guard exercised live); LS `final__*` tracks + `disagreements_resolved.json` (100 round-1 +disagreements annotated with final status, Δ, and intervention audit chains); calibration profile +mechanism (logistic/piecewise, `calibrated: true` path, unit-tested — fitting stays with the +US5 harness). + +**T037–T039 — validation harnesses**: determinism e2e **passing** in 6.2 s against the reference +run (hermetic, U3 off); golden-compat value-diff harness (needs two full-pipeline runs → GPU/Mac); +degradation-suite generator (5 variants + injected-span manifest, generated under +`artifacts/degradation_suite/`) + SC-001 checker. 12/12 unit + 1/1 e2e tests green; ruff clean +across all touched files. + +**Spec status: every task in tasks.md is now implemented.** Remaining full-env verification +(documented per task): comparator test suite green post-split, golden-compat diff on a GPU run, +degradation-suite pipeline runs, one-time MMS_FA download for live U3. + +## Sixth pass (2026-07-24) — architecture moves T046–T050 executed + +Per [architecture-review.md](./architecture-review.md): both parent `__init__`s are now PEP-562 lazy +(the entire `adaptive.loop` import chain loads with zero heavy modules in a bare env; the driver's +sys.modules stub hack is deleted); `adaptive/backends.py` is reduced to guarded task-API gateways — +U1 through `transcribe_audios(return_timestamps="word")`, consensus alignment through the NEW +`forced_alignment/mms_fa.py` (the previously dead torchaudio slot), overlap posteriors through +`FramePosterior.per_class`/`overlap_probs()` (FR-016 now lives in the task module), fine-hop +embeddings through the workflow extractor; `audio_io` replicates `prepare_audio` exactly for crop +cache-signature parity with enhancement via `enhance_audios`; transcript fusion is promoted to +`tasks/speech_to_text_ensemble/` (explicit `weights`); `ScriptLine.iter_leaves()`, +`normalize_transcript_for_wer`, jiwer-backed WER, and LS ground-truth parsing moved to their named +homes. + +**Why the DSP audio fallback stays (and how it's kept honest)**: it exists solely for artifact-driven +runs outside a senselab install — the mode this prototype was built and validated in. It is now never +silent: the loader used is recorded in `convergence.json → audio_backend` with the senselab-path +failure reason, and policy `audio_io_backend: senselab` makes production runs fail loudly instead +(same pattern for `u1_backend`). Verification: 15/15 unit + 3/3 votes tests, hermetic determinism e2e +(cache-replay-only policy) passing, ruff clean; run15 — executed through the relocated code with live +U1/I4 — reproduced every ground-truth metric exactly (fused WER 0.059/0.000, speaker accuracy 0.8095, +boundary 1.0/0.5). Sandbox-environment notes: the senselab loader/ASR paths could not be exercised +live here (torchaudio-2.13/torchcodec decode gap + 30 s cold imports on the degraded FS — both +environmental; repo pins torchaudio 2.8) — their fallback reasons are recorded in provenance, +demonstrating the envelope; validate the primary paths with the full-env checks below. + +## Integration fix (2026-07-24, after scene-spec US4 landed) + +Commit `581b18b3` (scene-quality-utterance US4) multiplies the utterance parquet's +`aggregated_uncertainty` by the FR-019 scene coupling (clipped, with the pure per-vote value +preserved on `raw_aggregated_uncertainty` and an auditable `__utterance_pre_coupling__` marker). +Since the coupled value is no longer a function of the votes alone, the adaptive belief store now +anchors on the **pre-coupling scale**: `parity_check` compares against `raw_aggregated_uncertainty` +(fallback to the legacy column on pre-FR-019 artifacts), loop thresholds/deltas stay on the per-vote +scale, and `disagreements_resolved.json` carries an explicit `scale_note`. Verified: 23/23 tests +(incl. US4's 8 new coupling tests) and raw-anchored parity ALL CLEAN (538/538 buckets) on the +reference artifacts. No changes needed in the US4 commits themselves. + +Coordination note: the branch now has two deliberate calibration surfaces — US4/US5's +scene/utterance calibration (`params["calibration"]`, quality.py convention) and the adaptive loop's +fused-word-confidence calibrator (`policy.calibration_profile`, logistic/piecewise). US5's profile +doc should name both so the fitting script can optionally emit a fused-confidence section. + +## Known limitations (tracked in tasks.md) + +- I4 overlap posteriors code-complete but unvalidated here (gated `pyannote/segmentation-3.0` needs an + HF token); with it, the irreducible region would be named `overlapping_speech` instead of the + generic reason, and the aleatoric floor would include the overlap term. +- U1's senselab-native backend routing (subprocess-venv ASR models) is follow-up; the live path here + uses the HF whisper pipeline with a policy-declared model pool. +- Short-segment identity resolution is bounded by the stored embedding hop (0.25 s ⇒ the 0.38 s GT + segment was not separated); full envs re-embed at `identity.fine_hop_s` (0.1 s). +- Word-level (not phoneme) pair distances in the sandbox fallback (`pair_distance_kind: "word"` — + g2p_en unavailable); full env uses `harvest_utterance_votes` unchanged. +- Fused confidences are uncalibrated (`calibrated: false`; T033) and depressed by slot fragmentation; + the *ordering* is what the localization result relies on. +- Stream election weights favored the enhanced stream (presence+quality over utterance agreement) — + a policy knob to revisit; fusion still scored best-in-ensemble on it. +- Triage round (US1) not exercised — inputs were pre-computed artifacts by design. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/quickstart.md b/specs/20260723-225523-dynamic-uncertainty-workflow/quickstart.md new file mode 100644 index 000000000..20de418ce --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/quickstart.md @@ -0,0 +1,73 @@ +# Quickstart: Uncertainty-driven adaptive analysis workflow + +## Run it + +```bash +# Default adaptive run (triage → baseline → ≤1 intervention round → fusion) +uv run python scripts/analyze_audio.py path/to/audio.wav + +# Legacy single-shot (golden-compat mode) +uv run python scripts/analyze_audio.py audio.wav --max-rounds 1 --enhancement always +``` + +## What to look at, in order + +1. `final/convergence.json` — `run_state`, per-axis converged/irreducible counts, budget, and + `next_actions` (what more budget would buy). +2. `final/transcript.json` — the fused answer: words with speaker, confidence, alternates. +3. `final/iterations.json` — every decision the loop made (fired, deferred, blocked, failed) with + trigger values and post-hoc uncertainty deltas. +4. `rounds//` — per-round belief parquets and region proposals when debugging a decision. +5. Existing artifacts (9 uncertainty parquets, LS bundle, `disagreements.json`, `timeline.png`) — + unchanged locations; the LS bundle gains `final__*` consensus tracks and + `final/disagreements_resolved.json` shows which round-1 disagreements the loop resolved. + +## Validation recipes + +```bash +# 1. Determinism (SC-004): identical runs → identical decision logs +uv run python scripts/analyze_audio.py audio.wav --output-dir /tmp/a +uv run python scripts/analyze_audio.py audio.wav --output-dir /tmp/b +diff <(jq -S . /tmp/a/*/final/iterations.json) <(jq -S . /tmp/b/*/final/iterations.json) # empty + +# 2. Golden compat (SC-005): legacy mode vs pre-feature golden run +uv run python scripts/analyze_audio.py audio.wav --max-rounds 1 --enhancement always \ + --no-adaptive-outputs --output-dir /tmp/compat +# then compare per-task JSONs + 9 parquets against the checked-in golden manifest + +# 3. Targeted improvement (SC-001): inject 3 s of noise, watch the loop attack it +uv run python - <<'PY' +import torch, torchaudio +w, sr = torchaudio.load("clean.wav"); s, e = int(5.0*sr), int(8.0*sr) +w[:, s:e] += 0.3*torch.randn_like(w[:, s:e]); torchaudio.save("degraded.wav", w, sr) +PY +uv run python scripts/analyze_audio.py degraded.wav --enhancement auto --max-rounds 3 +jq '[.entries[] | select(.status=="fired")] | map({rule, region_id, delta})' \ + /final/iterations.json # expect U1/U2/S1 on a region covering ~5–8 s + +# 4. Triage gates (SC-002/003) +uv run python scripts/analyze_audio.py clean.wav --enhancement auto # no enhanced_16k/ dir +uv run python scripts/analyze_audio.py silence.wav # run_state == "no_speech" +``` + +## Reading a region's story + +```bash +RID=r2_utterance_0 +jq --arg r "$RID" '.entries[] | select(.region_id==$r)' /final/iterations.json +python - <<'PY' +import pandas as pd +df = pd.read_parquet("/rounds/3/belief/utterance.parquet") +print(df[df.status != "open"][["start","end","aggregated_uncertainty","epistemic", + "aleatoric_floor","status","irreducible_reason","round"]]) +PY +``` + +An `irreducible: overlapping_speech` region with `aleatoric_floor ≈ residual` is the loop saying: +models disagree because two people are talking at once — more models will not fix this; separation +(v2 `--enable-overlap-separation`) or human review will. + +## Development order + +Phases in [plan.md](./plan.md): A (harvest/aggregate split, behavior-neutral) → B (triage + gating) → +C (loop core) → D (identity/overlap + fusion) → E (v2). Run `/speckit.tasks` to generate tasks.md. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/research-models-2026.md b/specs/20260723-225523-dynamic-uncertainty-workflow/research-models-2026.md new file mode 100644 index 000000000..8ac6cce98 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/research-models-2026.md @@ -0,0 +1,118 @@ +# Model landscape refresh (mid-2024 → mid-2026) — scene/events, diarization/VAD, enhancement + +**Date**: 2026-07-24 · **Method**: three parallel research agents (web + HF Hub verification of every +model id/license/gated flag; claims marked *unverified* where a primary source wasn't confirmed) · +**Anchors being reconsidered**: AST (2021) + YAMNet (2019); pyannote community-1 + Sortformer v1; +segmentation-3.0 + Brouhaha; ECAPA/ResNet embeddings; SepFormer-WHAM16k (2021). + +## Cross-cutting literature findings (they validate the adaptive-loop design) + +1. **"When De-noising Hurts" (arXiv 2512.17562, Dec 2025)**: enhancement preprocessing degraded WER in + **40/40** model×noise configurations (+1.1 to +46.6 abs). Modern ASR is internally noise-robust; + enhancement deletes acoustically useful detail. This is exactly the failure we measured on the test + clip (SepFormer deleted words) — and confirms three choices already implemented: the raw pass stays + authoritative; the fusion stream is chosen by utterance-evidence consistency, not by quality scores; + enhancement is a gated auxiliary stream (`--enhancement auto`, triage FR-003). +2. **URGENT 2025 (arXiv 2505.23212)**: best universal enhancer was *discriminative*; generative SE + shows hallucination/language-dependency risk. → Diffusion/LM restorers (Resemble, Sidon, LLaSE-G1, + SGMSE+) must never feed ASR by default; they belong behind an explicit restoration flag. +3. **ETH diarization benchmark (arXiv 2509.26177, Sep 2025)**: sub-0.5 s segments account for <5% of + DER; the dominant failures are boundary imprecision and **speaker-counting collapse** — precisely + our 4-speakers→1-cluster failure. Fix = better counting models, not finer frames alone. +4. **SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md** (repo handoff note) aligns: frame posteriors over + segmentized VAD, ~100 ms reporting grid, coarse taggers as priors only — the triage round + implemented today follows it; its "add TEN VAD backend" recommendation is seconded below. + +## Front 1 — scene analysis / event tagging (replace AST + YAMNet) + +| Rank | Model (verified) | Why | License | Surface | +|---|---|---|---|---| +| 1 | **`mispeech/ced-base`** (+ `-small/-mini/-tiny`) | 527-class AudioSet posteriors, mAP **50.0** (vs AST 45.9), **variable-length input** — scores 0.5–1 s windows directly, removing AST's 10.24 s floor (the D2 constraint "AST never on crops" disappears). CPU-fast at `-mini/-tiny`. | Apache-2.0 | HF transformers (`trust_remote_code`) | +| 2 | **`fschmid56/PretrainedSED`** (BEATs/ATST-Frame "strong"; `frame_mn10` 3.8M, `frame_mn06` 1.6M) | True **40 ms frame-level** event posteriors (447 AudioSet-Strong classes, PSDS1 46.5). The MobileNet heads are the YAMNet replacement (finer, better, MIT). | MIT | pip-from-GitHub, pinned release | +| 3 | `topel/ConvNeXt-Tiny-AT` (0.471 mAP, MIT) / EfficientAT `mn40_as` (0.483, license *unverified*) | Light alternates | MIT / ? | pip | + +Excluded: LALMs (Qwen-Audio etc.) — free text, not calibrated tag distributions. Nothing gated. + +**senselab integration**: add CED to the classification dispatcher (HF pipeline path, like AST); +PretrainedSED as a new backend module; keep the AudioSet→source-mass map (527-class compatible; the +447-class strong set needs a map extension). Adaptive-loop payoff: scene re-checks on crops at any +length; region-level `src_*` re-computation becomes a legal intervention. + +## Front 2 — diarization, VAD/OSD, speaker embeddings + +Diarization (short-turn/counting weighted): + +| Rank | Model (verified) | Key numbers | License / gated | +|---|---|---|---| +| 1 | **`BUT-FIT/diarizen-wavlm-large-s80-md-v2`** (DiariZen) | Best open speaker-counting (ETH: 4-spk 12.7 DER); AMI 14.0 / DIHARD3 14.5 | **CC-BY-NC-4.0** (weights; research OK, no commercial), ungated | +| 2 | **`nvidia/diar_streaming_sortformer_4spk-v2`** | ~4× better 4-spk than v1 (13.2 vs 21.3); exposes **T×4 @ 0.08 s frame posteriors** via `include_tensor_outputs=True` | **CC-BY-4.0**, ungated | +| 3 | `pyannote-community/speaker-diarization-community-1` | Current default — **ungated mirror exists** (the canonical repo is soft-gated); CPU-feasible | CC-BY-4.0 | +| — | `nvidia/diar_sortformer_4spk-v1` (current) | The model that failed our clip; also CC-BY-**NC** | retire from defaults | + +VAD / speech presence (triage-relevant): + +| Rank | Model | Why | License / gated | +|---|---|---|---| +| 1 | **`nvidia/Frame_VAD_Multilingual_MarbleNet_v2.0`** | **20 ms frame posteriors, 91.5K params, ONNX-on-CPU recipe, ungated** — the drop-in ungated alternative to gated segmentation-3.0 for the triage speech gate | NVIDIA OML (commercial per card), ungated | +| 2 | **`TEN-framework/ten-vad`** | 10/16 ms hop, per-hop probability, ~306 KB, arm64 — the backend the SPEECH_PRESENCE note asked for | Apache-2.0 | +| 3 | `snakers4/silero-vad` v6 | Ubiquitous cross-check; hangover lag makes it a co-voter, not primary | MIT | +| — | `pyannote/segmentation-3.0` (current) | Keep for **overlap** (no ungated frame-level OSD exists — verified gap) | MIT weights, **gated** | + +Speaker embeddings (change-point / short-segment — our 0.38 s failure): + +| Rank | Model | Why | License | +|---|---|---|---| +| 1 | **`IDRnD/ReDimNet`** (b0–b2, non-LM) | 1–4.7M params, Vox1-O 0.52% (b2); cheap enough for 0.1 s-hop sliding windows on CPU | MIT (torch.hub) | +| 2 | **ERes2NetV2** (`iic/speech_eres2netv2_sv_zh-cn_16k-common`) | The only model with *published* short-duration EER (0.98%@3s, 1.48%@2s) | Apache-2.0 (ModelScope/ONNX) | +| 3 | `speechbrain/spkrec-ecapa-voxceleb` (current) | Keep as A/B baseline | Apache-2.0 | + +Caveats (verified): **no embedding model is benchmarked <1 s** — 0.1–0.25 s hops are out-of-distribution +everywhere; rely on relative distances (as `identity_repair.py` does) and validate on labeled clips. +Avoid `-LM` checkpoints for short windows (tuned >3 s). + +## Front 3 — enhancement / separation / target-speaker extraction + +Enhancement (feeding ASR/diarization; identity-preservation weighted): + +| Rank | Model | Key numbers | License / surface | +|---|---|---|---| +| 1 | **`alibabasglab/MossFormerGAN_SE_16K`** | PESQ 3.57 / SI-SDR 20.6 (DNS2020) — highest signal-fidelity of the verified set; native 16 kHz | Apache-2.0, `pip install clearvoice` | +| 2 | **DeepFilterNet3** | Real-time CPU (RTF ~0.04); *filtering* (no resynthesis) → architecturally safest on clean speech | MIT/Apache dual, pip | +| 3 | `alibabasglab/FRCRN_SE_16K` | Light, proven (PESQ 3.24 DNS2020) | Apache-2.0, clearvoice | +| 4 | `wyz/tfgridnet_for_urgent24` | Trained/ranked on WAcc + SpkSim + phoneme similarity — the only family optimized against our exact axes | ESPnet | +| — | `speechbrain/sepformer-wham16k-enhancement` (current) | SI-SNR 13.8/PESQ 2.20; measured deleting words on clean speech | **retire as default** | + +Separation/TSE: `alibabasglab/MossFormer2_SS_16K` (2-spk separation, Apache-2.0); +`pyannote/speech-separation-ami-1.0` (PixIT — diarization-*aligned* per-speaker streams; MIT, gated); +`JusperLee/TIGER-speech` (0.82M params, CPU-feasible overlap repair); WeSep toolkit for +enrollment-based extraction (pairs with our unified clusters). Generative restorers (Resemble, Sidon, +LLaSE-G1) only behind an explicit restoration flag; Miipher-2 weights not released. + +**Do-no-harm gate (new, from the literature + our measurement)**: per-bucket +`SI-SDR(raw, enhanced)` + RMS-deletion detector inside VAD speech regions + DNSMOS delta +(ClearerVoice `SpeechScore` packages these) — route the enhanced stream per bucket only when it +demonstrably improves. This formalizes the S1 guard and the fusion-stream rule; optional +observation-adding (mix ~20–30% raw back) further reduces artifact damage (Iwamoto et al. 2022). + +## Adoption plan (phased, additive, registry/policy only — no behavior change until enabled) + +1. **Triage hardening (now)**: MarbleNet v2 / TEN-VAD as ungated frame-posterior backends in + `voice_activity_detection` beside segmentation-3.0 (SPEECH_PRESENCE item 1); triage prefers + whichever is available — removes the HF-token dependency from round 0. +2. **Enhancement swap (high value)**: add MossFormerGAN_SE_16K + DeepFilterNet3 to the enhancement + dispatcher; policy `enhancement_model`; SepFormer stays available for reproducibility. Implement the + SI-SDR/deletion do-no-harm gate as S1's quantitative guard. +3. **Diarization defaults**: add Streaming Sortformer v2 (CC-BY-4.0 + frame posteriors → replaces v1 + and feeds the identity axis directly); DiariZen behind a research flag (NC license); keep + community-1 via the ungated mirror. +4. **Embeddings for I1/I2**: ReDimNet-b0/b2 + ERes2NetV2 as additional `speaker_embeddings` backends; + policy `identity.fine_hop_s` path benefits immediately (targets the unresolved 0.38 s speaker). +5. **Scene refresh**: CED (all sizes) into classification; PretrainedSED frame heads as the new + fine-grid event voter; retire the AST-crops prohibition in contracts/region-reprocessing.md. +6. **Overlap repair (v2/U4)**: MossFormer2_SS_16K or TIGER for separation-based re-ASR; PixIT where + diarization-consistent streams are needed. + +Each addition follows the existing backend pattern (dispatcher + registry entry + pinned revision + +`ensure_hf_model`/local-files-only, constitution VI). License review flags: DiariZen weights (CC-BY-NC), +Sortformer v1 (CC-BY-NC — already in defaults today, worth an explicit review), MarbleNet (NVIDIA OML), +EfficientAT/M2D (license text unverified). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/research.md b/specs/20260723-225523-dynamic-uncertainty-workflow/research.md new file mode 100644 index 000000000..e71253f64 --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/research.md @@ -0,0 +1,162 @@ +# Research & Decision Record: Uncertainty-driven adaptive analysis workflow + +**Branch**: `20260723-225523-dynamic-uncertainty-workflow` | **Date**: 2026-07-23 + +Codebase findings this design is grounded in: + +- The pipeline is a single feed-forward pass; uncertainty is computed once and only reported + (`scripts/analyze_audio.py:1861-2264`, `compute.py:48-130`). No spec to date proposes acting on it — + compare-uncertainty explicitly defers ensemble-style refinement, and scene-quality-utterance's only + cross-signal element (`scene_quality_coupling`, FR-019) is a static one-directional multiplier. +- `compute_uncertainty_axes` interleaves expensive harvesting (embedding extraction per model per pass, + frame posteriors, Brouhaha, per-bucket g2p) with cheap pure aggregation, and mutates the caller's + `passes` dict (synthetic silhouette source, `compute.py:211-236`). There is no re-aggregate entry + point: changing only the aggregator re-runs all GPU inference. +- The content-addressable cache keys on the *audio signature* of the exact waveform + (`scripts/analyze_audio.py:741-806`), so cropped segments cache correctly with zero cache changes. + ASR and alignment caches are already independent (FR-024 of the asr-extensions spec). +- `frame_posteriors.py:75-88` computes segmentation-3.0's full powerset class posteriors and then + collapses them to a scalar P(speech) = `1 − P(no-speaker class)` (max over the class axis only as a + multilabel fallback); the per-class posteriors — including the overlap classes — are discarded before + return, so exposing an overlap posterior is a small additive extension. +- Existing one-directional couplings to preserve: silhouette voter injection, empirical cosine + calibration floors, presence intensity mask (`compute.py:211-236, 441-456, 405-418`). + +## D1 — Control: deterministic policy engine (not an LLM agent) + +**Decision**: The loop is driven by declarative rules: `trigger predicate over belief state → action → +cost class → guards → priority = expected_gain / cost`. Stable tiebreaks (priority, axis priority +utterance > identity > presence, region start). Policy is data (D10); the engine is a pure function of +(belief state, regions, budget, policy). + +**Rationale**: senselab's value proposition is reproducible pipelines; every existing artifact carries +provenance and cache keys. A stochastic planner would break SC-004 (byte-identical decision logs) and +make irreducibility claims unauditable. Rules are unit-testable without models. + +**Alternatives considered**: (a) LLM agent planner — flexible, but non-deterministic, unauditable, +and adds a serving dependency to a batch pipeline; rejected for the control path, noted as a possible +post-hoc *advisor* over `convergence.json` (out of scope). (b) Bandit/RL selection — needs reward +data we don't have; the expected-gain heuristics encode the same idea transparently. (c) Hybrid +(deterministic core + LLM for irreducible regions) — deferred with (a); the hook point is +`next_actions` in convergence.json. + +## D2 — Re-processing unit: cropped regions with padding and midpoint merge-back + +**Decision**: Interventions operate on `extract_segments` crops of `[start − pad, end + pad]` +(pad_s = 1.0 default), boundaries first quantized to the reporting grid, snapped outward to the nearest +presence trough (p_voice local minimum < 0.2) within the pad when one exists. Crop-local outputs are +offset-mapped to file time; only words/frames whose **midpoint lies inside the core region** merge back +into the vote store. Min-length rules: ASR crops ≥ 1.0 s post-pad; AST is never run on crops (10.24 s +native window, `scripts/analyze_audio.py:61-67`) — short-crop scene checks use YAMNet (0.96 s) and +frame posteriors. + +**Rationale**: Whole-file re-runs make per-region iteration unaffordable; crops bound cost by +uncertainty mass. Padding gives ASR acoustic context; midpoint merge-back avoids double-edged words; +grid quantization makes crop signatures repeatable → cache hits across rounds and runs. + +**Alternatives**: whole-file re-runs with different models only (cost scales with file, not with +uncertainty; rejected as the primary mechanism, still reachable as a heavy-class rule); VAD-segment +units (couples the unit to one signal — the thing we're uncertain about); fixed tiling (ignores +uncertainty structure). + +## D3 — Diarization stays whole-file; identity repair is local + +**Decision**: Never re-run diarizers on crops. Identity interventions are: fine-hop re-embedding + +calibrated change-point evidence (I1), overlap posterior (I4), and re-clustering with updated evidence. + +**Rationale**: Diarizers cluster globally; a crop severs speaker continuity and returns labels that +cannot be mapped back reliably. Embeddings already provide a diarization-independent, localizable +identity signal (`embeddings.py:91`, cosine calibration `embeddings.py:666`), and label unification +across models exists (`clustering.py:202`). + +## D4 — Enhancement: conditional whole-file + per-region stream election (no per-region enhancement in v1) + +**Decision**: `--enhancement auto` runs SepFormer once on the whole file iff triage finds degraded +speech (C1). Downstream, election (S1) picks per region which stream's evidence to trust and which +stream re-processing uses (C5), guarded: the enhanced stream cannot win where raw-side phonetic evidence +(PPG activity, ASR agreement on raw) contradicts enhanced-side speech — SepFormer can hallucinate +speech-like energy from noise. + +**Rationale**: Keeps the raw/enhanced pairing that the delta axes and existing contracts assume +(`raw_vs_enhanced` parquets), while removing the always-2× cost and adding the per-region choice the +one-shot pipeline can't make. Per-region enhancement creates boundary artifacts at crop seams and a +combinatorial pass space; deferred. + +## D5 — Vote merge semantics and family weights + +**Decision**: Vote key = (axis, bucket, model_id, stream, scope, round). Region-scoped votes **shadow** +file-scoped votes of the same (model_id, stream) in covered buckets (shadowed rows persist for +provenance, excluded from aggregation — most-specific-scope-wins, latest-round-wins within equal +scope). Distinct models/streams coexist. Aggregation weights each model by `1 / family_size` from a +policy-declared family map (whisper-derived = {openai/whisper-*, nyralabs/CrisperWhisper*}, nemo, +qwen, granite, ...), so intra-family agreement doesn't masquerade as independent evidence. + +**Alternatives**: averaging region+file votes of the same model (double-counts the same model on the +same audio); full covariance modeling between models (no data to fit it; family buckets are an honest +coarse prior). + +## D6 — Convergence thresholds and budget classes + +**Decision**: Reuse the established bins — θ_high = 0.66 (seed), θ_low = 0.33 (converged) — matching +disagreements/LS semantics. Improvement floor ε = 0.05 per intervention; `--max-region-rounds 2`; +budget classes: **light** (no model load: DSP, re-aggregation, adjudication over existing evidence), +**medium** (one model forward on one crop: fine posteriors, re-embed, one ASR on crop, re-align), +**heavy** (reserve-model load, separation, any whole-file re-run). Defaults: medium ≤ 24/run, +heavy ≤ 4/run, both overridable (FR-018). + +**Rationale**: consistent thresholds keep parquet/LS/loop semantics aligned; cost classes make budgets +hardware-portable where wall-clock caps aren't. + +## D7 — Epistemic vs aleatoric split, and irreducibility + +**Decision**: Per bucket: `epistemic` = the cross-source disagreement component (what more/better +evidence can reduce); `aleatoric_floor` = calibrated max of quality-driven floor (from SNR/clip/reverb +degradation) and overlap posterior (C7). A region is `irreducible` when interventions stop helping +(FR-017) **and** the floor explains the residual (residual ≤ floor + ε), with `irreducible_reason` +naming the dominant term; otherwise it exhausts budget as `budget_exhausted` (distinct status — honest +about "we stopped" vs "nothing more to learn"). + +**Rationale**: "Robust conclusion" requires saying *why* uncertainty remains. Overlap and SNR floors +are the two dominant, measurable aleatoric sources in this pipeline. + +## D8 — Harvest/aggregate split with a persistent VoteStore (prerequisite) + +**Decision**: Split `compute_uncertainty_axes` into `harvest_all(...) → VoteStore` (expensive, +model-touching) and `aggregate_all(VoteStore, policy) → axis results` (pure, cheap, covered-bucket +incremental). The synthetic silhouette source becomes store rows instead of a `passes` mutation. The +public signature is preserved by a thin wrapper. + +**Rationale**: Iteration is only affordable if re-scoring is free. Also fixes two flagged defects: +caller-visible mutation and no re-aggregate entry point (aggregator sweeps currently re-run all GPU +work). + +## D9 — Fusion: time-aligned word-level voting + consensus re-alignment + +**Decision**: Build word slots by grouping final-vote words across models with midpoint containment / +time-IoU ≥ 0.3; candidates weighted by family weight × model confidence (logprob / CTC / alignment +score when present); winner by weighted share, alternates kept when the margin is below a policy +threshold. Speaker attribution = unified cluster at word midpoint. Then C8: force-align the consensus +text once (Qwen aligner default, MMS fallback — same backends as the script) for authoritative +timestamps. Confidence calibration via the synthetic-harness profile when present (`calibrated: false` +otherwise). + +**Alternatives**: pick-single-best-model (discards the ensemble the pipeline already paid for); +lattice/confusion-network combination at decoder level (backends don't expose lattices uniformly). + +## D10 — Policy-as-data + +**Decision**: All thresholds, budgets, family weights, reserve models, and rule enable/disable live in +`adaptive/policy/default.yaml`; CLI flags override; `sha256(policy_yaml)` recorded in every round's +provenance and in `iterations.json`. + +## D11 — Failure envelope + +**Decision**: Each intervention runs inside the same failure envelope as the comparator +(`scripts/analyze_audio.py:2036-2040`): exceptions are caught, logged into `iterations.json` with +`status: "failed"`, the belief store is untouched for its buckets, and the loop continues. + +## D12 — Deferred to v2 + +Separation-based speaker-attributed re-ASR for overlap regions (U4/C11 — SepFormer separation + per-source +embedding matching + per-source ASR; expensive and needs its own validation), LID-gated re-ASR (U6), +LLM advisory hook over `convergence.json`, per-region enhancement, corpus-level policy learning. diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/spec.md b/specs/20260723-225523-dynamic-uncertainty-workflow/spec.md new file mode 100644 index 000000000..d5c2fe1aa --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/spec.md @@ -0,0 +1,363 @@ +# Feature Specification: Uncertainty-driven adaptive analysis workflow + +**Feature Branch**: `20260723-225523-dynamic-uncertainty-workflow` +**Created**: 2026-07-23 +**Status**: Draft +**Input**: User description: "analyze_audio should clean up the audio, detect environmental sounds, analyze quality, transcribe, and diarize — temporally; use individual temporal signals to improve other signals; use the signals to measure uncertainty about where speech exists, diarization, and word-aligned transcription; and run as a dynamic workflow that can iterate and come to a robust conclusion." + +## Context + +`scripts/analyze_audio.py` today is single-shot and exhaustive. `main()` runs every task on the raw +16 kHz audio (`run_pass`, `scripts/analyze_audio.py:1531`), unconditionally enhances the whole file and +repeats every task on the enhanced copy (`:1928-1949`), then calls the comparator once +(`compute_uncertainty_axes`, `compute.py:48`) to produce the three per-bucket uncertainty axes — +`presence`, `identity`, `utterance`. The axes are *reported* (9 parquets, `disagreements.json`, LS +tracks, `timeline.png`) but never *acted on*: the only consumer of high-uncertainty buckets is a human +reading `disagreements.json`. The couplings that do exist are one-directional and computed in a single +forward pass: embedding-cluster silhouette injected as a synthetic diarization voter +(`compute.py:211-236`), empirical cosine calibration floors flowing embeddings → identity +(`compute.py:441-456`), presence-derived `intensity_weight` masking identity/utterance rows +(`compute.py:405-418`), and the planned `scene_quality_coupling` multiplier (prior spec FR-019). + +Three structural consequences follow. (1) **Compute is spent uniformly, not where it matters**: clean +audio still pays for a full enhanced pass; a file with no speech still pays for four ASR models, two +diarizers, and alignment. (2) **Uncertainty is terminal**: a bucket where ASR models disagree stays +uncertain even when one more targeted model call, or a switch to the cleaner stream, would resolve it. +(3) **Known cross-signal repairs are impossible in one pass** — hallucination adjudication, diarization +boundary snapping to embedding change-points, overlap detection explaining joint identity+utterance +uncertainty, per-region raw-vs-enhanced stream election — because the signals that would drive each +repair are only available after the pass ends. + +This feature makes the three uncertainty axes the **control signal of a bounded, deterministic loop**: + +```text +round 0 round 1 rounds 2..K (K = --max-rounds) +TRIAGE → BASELINE EVIDENCE → [ TARGET → INTERVENE → RE-AGGREGATE → CONVERGE? ] +cheap signals conditional passes region proposal policy-ranked cheap, pure +(quality, frame (diarization, ASR, from belief actions on re-aggregation +posteriors, scene, alignment, PPG, state cropped regions over merged votes +openSMILE) embeddings, comparator) +``` + +Every round appends evidence (votes with full provenance) to a persistent **belief store**; aggregation +is re-run cheaply after each round; regions either converge (uncertainty ≤ θ_low), are proven +**irreducible** (uncertainty does not respond to intervention — with a machine-readable reason such as +`overlapping_speech` or `snr_floor`), or exhaust their budget. The final round fuses the accumulated +evidence into a **robust conclusion**: a consensus word-level transcript with calibrated confidences and +speaker attribution, a refined diarization, a fused presence track, and a complete audit trail of every +decision the loop made. The loop is driven by a deterministic, declarative policy engine — no LLM in the +control path — so identical inputs and policy produce identical decisions, in keeping with the +provenance/caching philosophy of the existing pipeline. + +## Signal-coupling matrix + +The core of "use individual temporal signals to improve other signals". *Existing* couplings are kept +and formalized; *new* couplings are introduced by this feature. Round = where the coupling first fires. + +| # | From (signal) | To (consumer) | Mechanism | Status | Round | +|---|---|---|---|---|---| +| C1 | Quality (SNR/C50/clip/bandwidth) + frame posteriors | Enhancement decision | `--enhancement auto`: run the enhanced pass only if some speech region is degraded | new | 0→1 | +| C2 | Frame posteriors (segmentation-3.0, Brouhaha) | Heavy-task gating | No confident speech anywhere ⇒ skip diarization/ASR/alignment/PPG, emit presence-only run | new | 0→1 | +| C3 | Quality per region | ASR vote weights | Per-region reliability prior on ASR votes (low SNR ⇒ down-weight fragile backends) | new | 2+ | +| C4 | Presence (p_voice) | Identity/utterance | `intensity_weight` mask (existing) + hard budget gate: no interventions on non-speech | existing+new | 1+ | +| C5 | Raw-vs-enhanced deltas + quality + presence | Stream election | Per-region choice of primary stream for re-processing; enhancement-artifact guard | new | 2+ | +| C6 | Embedding change-points (fine-hop) | Diarization boundaries | Snap disputed boundaries to calibrated cosine change-points; re-score identity | new | 2+ | +| C7 | Segmentation-3.0 per-class (powerset) posteriors | Identity + utterance | Overlap posterior explains joint uncertainty as aleatoric; routes overlap handling | new | 2+ | +| C8 | ASR consensus text | Forced alignment | Re-align consensus (not per-model) text ⇒ authoritative word timestamps | new | K | +| C9 | ASR+PPG+CTC agreement in "silent" buckets | Presence | Missed-speech correction vote where VAD said silence but phonetic evidence is strong | new | 2+ | +| C10 | Whisper no_speech/logprob + CTC + PPG + sources | Utterance + presence | Hallucination adjudication: purge hallucinated tokens from WER pairs and presence votes | new | 2+ | +| C11 | Diarization + overlap posterior | Utterance | Speaker-attributed re-ASR of overlap crops (v2, behind flag) | new (v2) | 3+ | +| C12 | Embedding clustering silhouette | Presence + identity | Synthetic diarization voter (`compute.py:211-236`) | existing | 1 | +| C13 | Per-pass empirical cosine calibration | Identity floors | `calibrate_cosine_uncertainty` band overrides CLI floors (`compute.py:441-456`) | existing | 1 | +| C14 | Scene sources (music/TV/machine masses) | Utterance + hallucination prior | `scene_quality_coupling` (prior spec FR-019) + raised hallucination prior in music regions | existing (planned) + new | 1, 2+ | + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Cost-aware triage and conditional processing (Priority: P1) + +A researcher runs the workflow on a batch of clinical recordings. Most are clean close-mic speech; a few +are silent (failed recording) or heavily degraded. On clean files the loop skips the enhanced pass +entirely; on silent files it stops after triage with a presence-only report; only degraded files pay for +enhancement and iteration. + +**Why this priority**: Immediate, measurable compute savings on every run, independent of the iteration +machinery; it exercises round 0 and the gating couplings (C1, C2) end-to-end. + +**Independent Test**: Run with `--enhancement auto` on (a) a clean clip, (b) a silent clip, (c) a +degraded clip. Verify (a) has no `enhanced_16k/` outputs, (b) stops after triage with +`run_state: "no_speech"` in `summary.json`, (c) runs both passes. + +**Acceptance Scenarios**: + +1. **Given** a clip whose triage SNR degradation stays below θ_enh in every speech region, **When** the + workflow runs with `--enhancement auto`, **Then** no enhancement model is loaded, no `enhanced_16k` + pass exists, and `iterations.json` records the decision with its trigger values. +2. **Given** a clip with frame-posterior P(speech) < 0.2 everywhere, **When** the workflow runs, **Then** + diarization/ASR/alignment/PPG never run, presence parquet is still emitted, and the run exits 0 with + `run_state: "no_speech"`. +3. **Given** `--enhancement always`, **Then** behavior matches today's unconditional two-pass run. + +--- + +### User Story 2 - Targeted re-processing of high-uncertainty regions (Priority: P1) + +After the baseline round, several 2–6 s regions show high utterance uncertainty (ASR models disagree). +The loop crops each region (with context padding), re-runs the ASR set on the region's elected stream, +escalates to a reserve model if disagreement persists, and re-aggregates. Regions where models now agree +converge; regions that don't respond are marked irreducible with a reason. + +**Why this priority**: This is the core promise of the dynamic workflow — uncertainty goes down where it +can, and is explained where it can't. + +**Independent Test**: Take a clean clip, inject localized noise into a 3 s span, run with +`--max-rounds 3`. Verify the injected span is proposed as a region, at least one intervention runs on +it, and either its utterance `aggregated_uncertainty` drops by ≥ ε or the region carries +`irreducible_reason`. + +**Acceptance Scenarios**: + +1. **Given** a bucket run of utterance uncertainty ≥ θ_high, **When** round 2 starts, **Then** a region + covering it appears in `rounds/2/regions.json` with axis, uncertainty mass, and elected stream. +2. **Given** a proposed utterance region, **When** the policy engine fires `U1_region_reasr`, **Then** + region-scoped ASR votes (scope `region:`) shadow that model+stream's file-scoped votes in covered + buckets, and re-aggregation changes only covered buckets. +3. **Given** an intervention whose re-aggregated uncertainty drop is < ε, **When** the round ends, + **Then** the region's `interventions_remaining` decrements, and after `--max-region-rounds` the region + is marked `irreducible` with the dominant explanation (e.g. `snr_floor`, `overlapping_speech`). +4. **Given** two identical runs (same audio, args, policy, warm cache), **Then** `iterations.json` is + byte-identical. + +--- + +### User Story 3 - Cross-signal repair (Priority: P2) + +The loop uses one signal to fix another: ASR text in a region the VAD calls silent is adjudicated +(hallucination vs missed speech) using no-speech probability, alignment CTC score, PPG activity, and +sound-source masses; diarization boundaries disputed between pyannote and Sortformer are snapped to +fine-hop embedding change-points; joint identity+utterance uncertainty over a region with high overlap +posterior is explained as overlapping speech rather than model failure. + +**Why this priority**: Cross-signal repair is what distinguishes iteration from merely "run more +models"; it reduces phantom disagreement (hallucinated tokens inflating WER pairs) and mislabeled +uncertainty (overlap read as model error). + +**Independent Test**: On a clip with music-only span where one ASR emits text, verify adjudication marks +those votes `hallucinated`, utterance WER pairs exclude them, and presence votes drop them. On a +two-speaker clip with a disputed boundary, verify the boundary evidence vote lands within the crop and +identity disagreement decreases. + +**Acceptance Scenarios**: + +1. **Given** ASR tokens in buckets with p_voice < 0.2, **When** `P3_hallucination_adjudication` fires and + ≥ 2 independent indicators support hallucination, **Then** affected votes are flagged, excluded from + utterance pairing (C10), and the verdict + indicator values are logged. +2. **Given** phonetically supported speech (PPG active, CTC score high, ≥2 ASR models agree) where frame + posteriors were low, **Then** a `adjudicator/missed_speech` presence vote is added (C9) and presence + uncertainty in those buckets increases or the belief flips — never silently. +3. **Given** identity uncertainty at a diarization boundary and a clear fine-hop cosine change-point, + **Then** an `embedding_changepoint/` evidence vote supports the nearest model boundary and + re-aggregated identity uncertainty in the region decreases. +4. **Given** a region whose overlap posterior mean ≥ 0.5, **Then** identity/utterance rows in it gain + `aleatoric_floor` ≥ overlap posterior and, if uncertainty stays high, `irreducible_reason = + "overlapping_speech"`. + +--- + +### User Story 4 - Robust conclusion with audit trail (Priority: P2) + +When the loop stops, the researcher gets a single fused answer, not nine parquets to reconcile: a +word-level consensus transcript (text, start, end, speaker, calibrated confidence, alternates where +contested), a final diarization with boundary confidence, a fused presence track, and `iterations.json` +recording every rule that fired, why, what it cost, and what it changed. + +**Why this priority**: "Come to a robust conclusion" is the user-facing deliverable; everything else is +machinery. + +**Independent Test**: Run on a tutorial clip; verify `final/transcript.json` words are monotone in time, +every word carries speaker + confidence ∈ [0,1], contested words list alternates, and every entry in +`iterations.json` references trigger values present in the belief store. + +**Acceptance Scenarios**: + +1. **Given** a completed run, **Then** `final/` contains `transcript.json`, `diarization.json`, + `presence.parquet`, `convergence.json`, and `iterations.json` conforming to + `contracts/final-outputs.md`. +2. **Given** a word where ≥ 2 model families agree, **Then** its confidence exceeds that of any + single-family word, and family weights prevent double counting (two Whisper-derived models ≠ two + independent votes). +3. **Given** contested words, **Then** alternates carry their vote shares and the LS bundle shows the + final consensus track alongside the existing per-model tracks. + +--- + +### User Story 5 - Bounded, budgeted convergence (Priority: P3) + +An operator caps compute: `--max-rounds`, per-round and total intervention budgets by cost class, and +per-region intervention caps. The loop reports what it spent, what converged, and what it would have +done next with more budget. + +**Why this priority**: Makes the loop safe to run unattended and its cost predictable. + +**Independent Test**: Run with a tiny budget (`--budget-medium 2 --budget-heavy 0`) on a degraded clip; +verify the loop stops within budget, `convergence.json.budget` accounts for every spent unit, and +`next_actions` lists the top unfunded interventions. + +**Acceptance Scenarios**: + +1. **Given** exhausted medium budget, **Then** pending medium interventions defer, appear in + `next_actions`, and the loop proceeds to fusion. +2. **Given** `--max-rounds 1`, **Then** no interventions run and outputs are the baseline single-shot + set (see FR-024 compatibility). +3. **Given** any run, **Then** Σ logged costs in `iterations.json` = `convergence.json.budget.spent`. + +## Requirements *(mandatory)* + +### Loop control & rounds + +- **FR-001**: The workflow MUST run as ordered rounds: triage (0), baseline evidence (1), intervention + rounds (2..K, K = `--max-rounds`, default 3), fusion (final). Rounds MUST be skippable by + configuration but not reorderable. +- **FR-002**: Triage MUST run only light signals (quality DSP, Brouhaha, segmentation-3.0 posteriors, + AST/YAMNet, openSMILE) and MUST reuse the existing content-addressable cache keys (same task names, + params) so triage results are shared with the baseline round. +- **FR-003**: With `--enhancement auto`, the enhanced pass MUST run only if triage finds at least one + region with speech posterior ≥ θ_speech and quality degradation ≥ θ_enh. `always` and `never` MUST be + supported; `--no-enhancement` remains an alias for `never`. +- **FR-004**: If triage finds no bucket with P(speech) ≥ θ_speech, the run MUST stop after emitting + presence outputs, with `run_state: "no_speech"` and exit code 0. + +### Belief store & aggregation + +- **FR-005**: All model evidence MUST be recorded as votes in a persistent belief store + (`contracts/belief-store.md`), keyed by (axis, bucket, source) where source = (model_id, stream, + scope, round). Existing comparator votes are the round-1 population of this store. +- **FR-006**: Aggregation MUST be re-runnable from the belief store alone, without model inference + (harvest/aggregate split). Re-aggregation after an intervention MUST touch only buckets covered by new + votes. +- **FR-007**: Vote merge semantics: a region-scoped vote from (model, stream) MUST shadow the same + (model, stream)'s file-scoped vote in covered buckets; votes from distinct models or streams coexist. + Shadowed votes remain in the store (provenance), excluded from aggregation. +- **FR-008**: Cross-model aggregation MUST apply model-family weights (declared in policy) so that + same-family models (e.g. Whisper-derived) do not double-count as independent evidence. +- **FR-009**: Each belief row MUST carry: round of last update, status ∈ {open, converged, irreducible, + budget_exhausted}, elected stream, epistemic and aleatoric components, and (when irreducible) a + machine-readable `irreducible_reason`. + +### Region proposal & interventions + +- **FR-010**: After each aggregation, contiguous high-uncertainty regions MUST be proposed per axis: + seed at ≥ θ_high (default 0.66), expand while ≥ θ_low (default 0.33), merge gaps < g (default 0.5 s), + pad ± pad_s (default 1.0 s) snapped outward to presence troughs where available, capped at top-N + (default 8) by uncertainty mass per round. +- **FR-011**: Interventions MUST be declared as data (trigger predicate over belief state, action, cost + class, guards, priority function) per `contracts/interventions.md`; the engine MUST rank runnable + interventions by expected-gain/cost and execute within budget (`contracts/policy-engine.md`). +- **FR-012**: The v1 catalog MUST include at minimum: P2 fine-grid posterior re-analysis, P3 + hallucination adjudication, C9 missed-speech correction, U1 region re-ASR on elected stream, U2 + reserve-model escalation, U3 consensus re-alignment, I1 boundary refinement via embedding + change-points, I4 overlap detection via per-class segmentation posteriors, and S1 stream election. +- **FR-013**: Region re-processing MUST crop with context padding via `extract_segments`, map timestamps + back to file time, and merge back only words/frames whose midpoint lies in the core (unpadded) region + (`contracts/region-reprocessing.md`). +- **FR-014**: Cropped-region model calls MUST flow through the existing cache (`cache_key` on the crop's + own audio signature), so repeated runs and overlapping regions replay cached results. +- **FR-015**: Stream election (C5) MUST pick, per region, the stream whose belief scores best (presence + confidence, quality, utterance agreement), MUST apply the enhancement-artifact guard (reject enhanced + when raw-side phonetic evidence contradicts enhanced-side speech), and MUST record the election and its + inputs. +- **FR-016**: Segmentation-3.0 per-class (powerset) posteriors MUST be exposed alongside the existing + collapsed P(speech) (extension of `frame_posteriors.py:78-88`), yielding an overlap posterior used by + I4 and the aleatoric floor (C7). + +### Convergence & budget + +- **FR-017**: A region converges when its axis uncertainty ≤ θ_low; it becomes irreducible when an + intervention improves it by < ε (default 0.05) and `--max-region-rounds` (default 2) is exhausted, or + when its aleatoric floor ≥ its uncertainty. `irreducible_reason` MUST name the dominant floor + (`overlapping_speech`, `snr_floor`, `non_speech_vocalization`, `single_model_coverage`, ...). +- **FR-018**: Budgets MUST be enforced per cost class (light/medium/heavy) per round and per run; + exceeding budget defers interventions into `convergence.json.next_actions` rather than dropping them + silently. +- **FR-019**: The loop MUST terminate: hard cap `--max-rounds`; a round with zero fired interventions + ends iteration early. +- **FR-020**: Every policy decision (fired or deferred) MUST be logged to `iterations.json` with + trigger values, cost, and post-hoc uncertainty delta (`contracts/final-outputs.md`). + +### Fusion & outputs + +- **FR-021**: Fusion MUST produce `final/transcript.json` by time-aligned word-level voting over the + final vote set (family-weighted, confidence-weighted), with speaker attribution from the unified + clustering, alternates for contested slots, and confidences calibrated via the synthetic harness + (prior spec US5) when a calibration profile exists — otherwise raw vote shares, flagged + `calibrated: false`. +- **FR-022**: Fusion MUST produce `final/diarization.json` (unified clusters, refined boundaries with + confidence), `final/presence.parquet` (fused p_voice + status), and `final/convergence.json` + (per-axis convergence counts, uncertainty mass per round, budget accounting, irreducible regions with + reasons). +- **FR-023**: The consensus transcript and final presence/diarization MUST be attached to the LS bundle + as additional tracks; existing per-model tracks are unchanged. + +### Compatibility, determinism, failure + +- **FR-024**: `--max-rounds 1 --enhancement always` MUST reproduce today's outputs: the existing 9 + uncertainty parquets, per-task JSONs, LS bundle, disagreements.json and summary.json keys are + unchanged (new keys/files strictly additive). Default flag values MUST preserve current behavior for + the existing artifact set. +- **FR-025**: The loop MUST be deterministic given (audio, args, policy file, senselab version): stable + intervention ordering (priority, then axis priority utterance > identity > presence, then region + start), seeded clustering, and a policy hash recorded in provenance. +- **FR-026**: Any intervention failure MUST be caught, logged with the exception, leave the belief store + unchanged for its buckets, and not abort the run (mirrors the comparator's failure envelope, + `scripts/analyze_audio.py:2036-2040`). +- **FR-027**: All thresholds (θ_speech, θ_enh, θ_low, θ_high, ε, g, pad_s, N, budgets, family weights, + reserve models) MUST live in a versioned policy file with CLI overrides — no hardcoded parameters + (constitution VIII). + +## Success Criteria *(mandatory)* + +- **SC-001**: On a validation suite of clips with localized injected degradation (noise, clip, reverb + spans), ≥ 70% of injected spans are proposed as regions in round 2, and ≥ 50% of them either reduce + utterance uncertainty by ≥ 0.15 or carry a correct `irreducible_reason` by loop end. +- **SC-002**: On clean clips, `--enhancement auto` skips the enhanced pass and total wall-clock is + ≤ 60% of today's two-pass run (warm cache excluded from measurement). +- **SC-003**: On silent clips, the run stops after triage in ≤ 15% of the full-run wall-clock. +- **SC-004**: Two consecutive runs with identical inputs and warm cache produce byte-identical + `iterations.json` and `final/convergence.json`. +- **SC-005**: `--max-rounds 1 --enhancement always` output passes the existing regression suite with no + changes to pre-existing artifacts (schema and values), verified by diffing a golden run. +- **SC-006**: Hallucination adjudication removes ≥ 80% of ASR tokens injected into music-only spans of + the validation suite from utterance pairing, with ≤ 5% false-purge of true speech tokens. +- **SC-007**: Fused word confidences are monotonically related to correctness on the synthetic suite + (higher-confidence deciles have lower WER); with a calibration profile, ECE ≤ 0.10. +- **SC-008**: Every intervention in `iterations.json` on the validation suite links to trigger values + reproducible from the persisted belief store (audit-trail completeness = 100%). + +## Out of Scope + +- Streaming / online processing; the loop operates on complete files. +- LLM-driven planning or an agentic controller in the control path (a post-hoc advisory hook over + `convergence.json` may be added later; explicitly not in v1). +- Human-in-the-loop actions mid-run; Label Studio remains post-hoc review. +- Model training / fine-tuning; per-model ensemble or MC-dropout uncertainty (already deferred by the + compare-uncertainty spec). +- Separation-based speaker-attributed re-ASR of overlap regions (C11/U4): specified as v2, implemented + behind a default-off flag only if time permits. +- Changing the existing 9-parquet contract, LS track names, or per-task JSON shapes (additive only). +- Corpus-level adaptation across files (each run is independent). + +## Key Entities + +See [data-model.md](./data-model.md): VoteStore, BeliefRow, Region, InterventionRecord, StreamElection, +RoundSummary, ConvergenceReport, FinalWord. + +## Assumptions & Dependencies + +- Builds on the merged comparator (compare-uncertainty spec) and the implemented US1–US3 of + scene-quality-utterance (quality columns, sound sources, frame posteriors, per-axis grids). The + utterance rework (US4: token logits) and calibration harness (US5) are pending there; this feature + degrades gracefully without them (P3 uses fewer indicators; FR-021 flags `calibrated: false`). +- `extract_segments` (`senselab.audio.tasks.preprocessing`) provides cropping; content-addressable cache + keys already incorporate the crop's audio signature, so region-level caching needs no cache change. +- Diarization remains whole-file (global speaker continuity); boundary repair is local via embeddings + (research.md D3). AST cannot run on crops < 10.24 s — short-crop scene checks use YAMNet and frame + posteriors (research.md D2). diff --git a/specs/20260723-225523-dynamic-uncertainty-workflow/tasks.md b/specs/20260723-225523-dynamic-uncertainty-workflow/tasks.md new file mode 100644 index 000000000..ad1bb62aa --- /dev/null +++ b/specs/20260723-225523-dynamic-uncertainty-workflow/tasks.md @@ -0,0 +1,166 @@ +# Tasks: Uncertainty-driven adaptive analysis workflow + +**Input**: Design documents from `/specs/20260723-225523-dynamic-uncertainty-workflow/` +**Prerequisites**: plan.md, spec.md, research.md (D1–D12), data-model.md, contracts/ + +**Implementation status (2026-07-24)**: all tasks below are implemented on branch +`20260722-175022-scene-quality-utterance` under `src/senselab/audio/workflows/audio_analysis/` +(`votes.py` + `adaptive/`) and `scripts/{analyze_audio,adaptive_loop,make_degradation_suite}.py`. +The loop is **artifact-driven**: `scripts/adaptive_loop.py` runs over a completed analyze_audio run +dir + the content-addressable cache; in-process invocation from analyze_audio itself (the full +`contracts/cli.md` flag set) is tracked as Phase 8 follow-up T040. An independent implementation +audit (2026-07-24) confirmed every checked claim against the code; its remaining gaps are enumerated +in Phase 8. + +## Phase 1: Setup + +- [X] T001 Create `adaptive/` subpackage skeleton per plan.md (`__init__.py`, module files, `policy/default.yaml`) +- [X] T002 [P] Policy loader with defaults-merge + `policy_hash` (`adaptive/policy.py`) +- [X] T003 [P] Light-import bootstrap for minimal environments (driver-level stub of heavy package `__init__`, `scripts/adaptive_loop.py`) + +## Phase 2: Foundational (blocking) — belief store (Phase A of plan.md) + +- [X] T004 Vote model + VoteStore with shadow/coexist/purge semantics per contracts/belief-store.md (`adaptive/belief.py`) +- [X] T005 Ingest existing run artifacts: 6 per-pass uncertainty parquets → votes + (`adaptive/belief.py`); per-model ASR/alignment/diarization JSONs → word streams + (`adaptive/interventions.load_outcomes_dir` + `adaptive/fusion.collect_word_streams`) +- [X] T006 Re-aggregation from votes via the existing pure aggregators (`aggregate_presence/identity/utterance`), incremental over covered buckets (FR-006) +- [X] T007 Round-1 parity check: re-aggregated values reproduce stored `aggregated_uncertainty` (harvest/aggregate split proof, D8) +- [X] T008 Harvest/aggregate split inside the comparator: NEW pure module `votes.py` + (`PassHarvest`, `aggregate_pass`, `compute_pass_deltas` — stdlib-light, unit-tested in + `votes_test.py`) + `compute.py` refactored to `harvest_pass` (model-touching) with + `compute_uncertainty_axes` as a thin wrapper. Aggregation math moved verbatim; caller-dict + mutation now opt-out via `mutate_passes=False` (default True preserves the timeline-plot + contract byte-for-byte). Verify on full env: `uv run pytest src/tests/audio/workflows/audio_analysis/` +- [X] T009 In-process vote-store integration point: `VoteStore.from_harvests(...)` consumes + `PassHarvest` objects directly (no parquet round-trip); the parquet ingest path remains for + artifact-driven runs. Round persistence stays with the loop driver by design. + +## Phase 3: US1 — Cost-aware triage & conditional processing (P1) + +- [X] T010 [US1] Triage round 0: pure decision module (`adaptive/triage.py` — frame-posterior speech + gate at ~100 ms aggregation per SPEECH_PRESENCE_CERTAINTY_ANALYSIS.md, Brouhaha SNR with + posterior-masked DSP fallback) + `run_triage` in `scripts/analyze_audio.py`; validated live with + segmentation-3.0 on 4 cases (clean/conversation/silent/noise-injected) +- [X] T011 [US1] `--enhancement {auto,always,never}` + `--triage-*` thresholds + no-speech early exit + (skips diarization/ASR/alignment/PPG, `run_state: "no_speech"`, `triage.json` provenance) in + `scripts/analyze_audio.py`; `--no-enhancement` kept as alias; default `always` preserves golden compat +- [X] T012 [US1] `run_pass` decomposed into `_stage_{diarization,scene,features,asr,alignment,ppg}` + (pure code motion; task names/params — hence cache keys — byte-identical). NOTE: any edit to + `scripts/analyze_audio.py` rotates `wrapper_version_hash` (whole-file sha256 by design), so the + first post-merge run re-populates the cache once. +- [X] T013 [US1] Budget ledger with light/medium/heavy classes, per-run caps, deferral → `next_actions` (FR-018) (`adaptive/policy.py`) + +## Phase 4: US2 — Targeted re-processing (P1) 🎯 prototype MVP + +- [X] T014 [US2] Region proposal: seed ≥ θ_high, expand ≥ θ_low, gap-merge, pad, grid-quantize, top-N by uncertainty mass (FR-010) (`adaptive/regions.py`) +- [X] T015 [US2] Deterministic policy engine: trigger→rank→admit within budget, stable total order (FR-011, FR-025) (`adaptive/policy.py`) +- [X] T016 [US2] U2 reserve escalation via **cache replay** (policy `reserve_asr_models`, default + whisper-large-v3-turbo, replayed from the content-addressable cache; re-harvest with + `harvest_utterance_votes`, region-scoped votes, family weights) (`adaptive/interventions.py`) +- [X] T017 [US2] Region-scope shadowing of file-scope votes from same (model, stream) on covered buckets (D5) +- [X] T018 [US2] U1 region re-ASR with live backends on cropped audio (`adaptive/audio_io.py` + `adaptive/backends.py`; HF whisper pipeline path with policy `u1_asr_models`; enhanced stream regenerated on demand via SepFormer with recorded raw fallback). Senselab-native backend routing (subprocess venvs) remains follow-up. +- [X] T019 [US2] Convergence marks: converged / irreducible / budget_exhausted + ε-monotonicity + max-region-rounds (FR-017) (`adaptive/convergence.py`) +- [X] T020 [US2] `iterations.json` decision log incl. deferred/blocked entries with trigger values (FR-020) + +## Phase 5: US3 — Cross-signal repair (P2) + +- [X] T021 [US3] S1 stream election per region with recorded scores; enhancement-artifact guard degraded to available evidence (FR-015) (`adaptive/interventions.py`) +- [X] T022 [US3] P3 hallucination adjudication over existing evidence (available indicators: native word confidence, source masses, presence p_voice); purge semantics on both axes (C10) +- [X] T023 [US3] C9 missed-speech correction vote (`adjudicator/missed_speech`) where phonetic/text evidence contradicts low p_voice +- [X] T024 [US3] I1 boundary refinement (`adaptive/identity_repair.py`): consensus adjacent-cosine change-point trajectory from stored per-window embeddings (live fine-hop re-embedding available in `backends.embed_windows` for full envs); boundary-confidence from prominence +- [X] T024a [US3] I2 re-cluster: change-point + diar-boundary segmentation, p_voice-weighted pooling, deterministic average-linkage cosine clustering, cross-model co-association consensus; refined clusters drive fusion speaker attribution + `final/diarization.json`; per-bucket `__cross_diar_label_disagreement__` recomputed with the new voter +- [X] T025 [US3] I4 overlap posterior via per-class segmentation-3.0 (`backends.overlap_posteriors`) — code-complete; gated-model validation pending (requires HF token; guards to `next_actions` otherwise) +- [X] T026 [US3] Aleatoric floor = max(quality, overlap posterior) in `belief._decompose` (overlap term populated by I4 when available) + +## Phase 6: US4 — Robust conclusion (P2) + +- [X] T027 [US4] Word-stream extraction from all ASR sources (native chunks + MMS/Qwen alignment words) (`adaptive/fusion.py`) +- [X] T028 [US4] Time-aligned word-slot voting with family weights + confidences; alternates below margin (FR-021, D9) +- [X] T029 [US4] Speaker attribution from unified identity clusters at word midpoint; `final/transcript.json` + segments rollup +- [X] T030 [US4] `final/diarization.json`, `final/presence.parquet`, `final/convergence.json` (FR-022) +- [X] T031 [US4] U3 consensus re-alignment (`backends.consensus_align` via torchaudio MMS_FA; + SIGALRM-guarded with policy `fusion.consensus_alignment{,_timeout_s}`; fallback = weighted member + timestamps recorded in `transcript.json.timestamps`). Guard validated live (timeout path); + full alignment runs after the one-time bundle download on a full env. +- [X] T032 [US4] `adaptive/ls_final.py`: `final__consensus_transcript(+__text)`, `final__diarization`, + `final__presence` LS tracks (additive copies under `final/`) + `disagreements_resolved.json` + (round-1 entries annotated with final status, Δ, and touching intervention ids). Validated on run14 + (21 word regions, 100 resolved entries). +- [X] T033 [US4] Calibration mechanism: `fusion.load_calibrator` (logistic / piecewise profiles), + policy `calibration_profile`, `calibrated: true` flag path; unit-tested. Fitting the profile itself + remains with the synthetic harness (scene-quality-utterance US5). + +## Phase 7: Validation & evaluation + +- [X] T034 [P] Ground-truth evaluation harness vs Label Studio export (presence acc, transcript WER fused-vs-per-model, cluster↔speaker mapping, boundary-uncertainty check, untranscribed-region check) (`adaptive/evaluate.py`) +- [X] T035 [P] End-to-end prototype run on `audio_48khz_mono_16bits` run dir + `updated-label-a7a37522.json` +- [X] T036 Unit tests for pure parts: shadowing, region proposal, plan ordering determinism, fusion voting (`src/tests/audio/workflows/audio_analysis/adaptive/adaptive_prototype_test.py`) +- [X] T036a Visual round-by-round timeline `final/timeline.png` — GT vs presence/identity/utterance (round-1 vs final overlay), regions, fired interventions with Δ, irreducible hatching, confidence-colored fused words (`adaptive/plot.py`) +- [X] T037 Determinism e2e (`adaptive_e2e_test.py::test_t037_determinism_byte_identical`, + env-gated on `SENSELAB_ADAPTIVE_E2E_RUN_DIR`; hermetic — U3 network path disabled) — PASSING + against the reference run dir (SC-004). +- [X] T038 Golden-compat harness (`test_t038_golden_compat_preexisting_artifacts`, env-gated on + `SENSELAB_GOLDEN_RUN_DIR`/`SENSELAB_CANDIDATE_RUN_DIR`): value-equality over the 9 uncertainty + parquets + per-task result payloads (SC-005). Requires two full-pipeline runs → execute on GPU/Mac. +- [X] T039 Degradation suite: `scripts/make_degradation_suite.py` (noise/clip/lowpass/silence/music + variants + injected-span manifest; suite generated under `artifacts/degradation_suite/`) + + `test_t039_injected_spans_attacked_or_explained` (SC-001 ≥ 70% attacked-or-explained; env-gated). + Full-pipeline variant runs execute on GPU/Mac. + +## Phase 8: Follow-ups from the 2026-07-24 implementation audit + +- [ ] T040 In-process adaptive integration in `scripts/analyze_audio.py` per contracts/cli.md: + `--max-rounds/--policy/--budget-*/--max-region-rounds/--region-top-n/--reserve-asr-models/` + `--enable-overlap-separation/--no-adaptive-outputs`, in-run `rounds/` + `final/` emission via + `VoteStore.from_harvests`, `summary.json` `adaptive` block, and the `--skip comparisons` warning. +- [ ] T041 `P2_fine_posteriors` rule (fine-hop posterior re-analysis on crops) — also unblocks I4's + contract dependency ("else fires P2 first"). +- [ ] T042 Final-output schema completion vs contracts/final-outputs.md: `final/diarization.rttm`, + `diarization.json` `member_labels`/`overlap`, `transcript.json.language`, presence.parquet contract + columns (`presence_confidence`, `elected_stream`, `overlap_posterior`). +- [ ] T043 Execute T038 (golden vs candidate full-pipeline runs) and T039 (degradation-suite pipeline + runs) on a GPU/Mac environment; record results in prototype-results.md. +- [ ] T044 Exercise `VoteStore.from_harvests` (unit test + first in-process caller, with T040). +- [ ] T045 Align contracts/interventions.md with implementation: U2 cost class (medium in code vs + heavy in contract) + family-majority guard; document `I2_recluster` as a catalog addition; U3 as a + fusion-stage step rather than a RULES entry; `max_region_rounds` enforced via convergence marks. + +### Architecture follow-ups (see [architecture-review.md](./architecture-review.md); T046–T050 implemented 2026-07-24) + +- [X] T046 Lazified `audio/workflows/__init__.py` + `audio_analysis/__init__.py` (PEP-562, incl. new + `harvest_pass`/`PassHarvest`/`aggregate_pass` exports); `_ensure_light_importable` stub deleted — + verified: the full `adaptive.loop` import chain loads with ZERO heavy modules in a bare env. +- [X] T047 `adaptive/backends.py` dissolved into guarded task-API gateways: U1 → + `speech_to_text.transcribe_audios` (`return_timestamps="word"`; policy `u1_backend: + auto|senselab|pipeline`, backend recorded in iterations.json); `consensus_align` → NEW + `forced_alignment/mms_fa.py` (fills the dead torchaudio slot); `overlap_posteriors` → + `FramePosterior.per_class` + `overlap_probs()` via `extract_speech_frame_posteriors(..., + include_per_class=True)` (closes T041's FR-016 dependency); `embed_windows` → the workflow's + `extract_per_window_embeddings` on crops. +- [X] T048 `adaptive/audio_io.py` routes through `tasks/preprocessing` (exact `prepare_audio` + replication ⇒ crop `audio_signature`/cache parity) and `tasks/speech_enhancement.enhance_audios`; + DSP loader retained as the labeled artifact-driven-outside-senselab fallback, **never silent**: + loader recorded in `convergence.json → audio_backend`, policy `audio_io_backend: + auto|senselab|dsp` ("senselab" = strict fail-loudly for production). +- [X] T049 (partial) `ScriptLine.iter_leaves()` added (call-site migration opportunistic); + `adaptive/evaluate` WER → `speech_to_text_evaluation.calculate_wer` with Levenshtein fallback; + canonical `normalize_transcript_for_wer` moved to `speech_to_text_evaluation/utils.py` + (aggregate.py re-exports the historical name); `load_ls_ground_truth` → + `audio_analysis/labelstudio.py`. REMAINING: `triage.dsp_snr_series` → `quality_control/metrics.py` + deferred until that module's pure-DSP/model-based split (it imports VAD+diarization at top today). +- [X] T050 Transcript fusion promoted to `tasks/speech_to_text_ensemble/` (`fuse_word_streams` with + explicit `weights`, `load_calibrator`, `iter_word_leaves`); `adaptive/fusion.py` keeps the + policy→weights wrapper + artifact collection + final-output writers. +- [ ] T051 Cache/provenance layer out of the script → `utils/tasks/cached_inference.py`; `_stage_*` + functions → `workflows/audio_analysis/stages.py`; `wrapper_version_hash` re-scoped to the stage + modules (documented invalidation-semantics change). Precondition for T040. (Own PR per review.) +- [ ] T052 Typed adaptive internals per house style: `adaptive/types.py` dataclasses (`Region`, + `PlannedIntervention`, `LoopContext`, `InterventionRule`) replacing dict soup; planner + regions + first. (Opportunistic, own PR.) + +## Dependencies + +Phase 2 blocks everything. US2 (Phase 4) is the MVP and depends only on Phase 2. US3 rules plug into +the same engine. US4 consumes the final belief state. T008/T012 (full-pipeline integration) are the +bridge from prototype to production and precede T010/T011/T018/T024/T025/T031/T032. diff --git a/src/senselab/audio/tasks/features_extraction/sparc.py b/src/senselab/audio/tasks/features_extraction/sparc.py index 3baac4fee..88d2f81c3 100644 --- a/src/senselab/audio/tasks/features_extraction/sparc.py +++ b/src/senselab/audio/tasks/features_extraction/sparc.py @@ -9,7 +9,7 @@ import subprocess import tempfile from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import numpy as np import torch @@ -183,7 +183,7 @@ def extract_sparc_features( lang: Optional[Language] = None, device: Optional[DeviceType] = None, resample: Optional[bool] = False, - ) -> List[Dict[str, torch.Tensor]]: + ) -> List[Dict[str, Any]]: """Extract SPARC articulatory features from audios. The SPARC model runs in an isolated subprocess venv with its own @@ -197,7 +197,9 @@ def extract_sparc_features( Returns: List of feature dicts with keys: ema, loudness, pitch, - periodicity, pitch_stats, spk_emb, ft_len. + periodicity, pitch_stats, spk_emb, ft_len. All values are tensors + except ``ft_len``, which is a plain scalar — hence the ``Any`` + element type rather than ``torch.Tensor``. """ device, _ = _select_device_and_dtype( user_preference=device, compatible_devices=[DeviceType.CUDA, DeviceType.CPU] @@ -262,8 +264,11 @@ def extract_sparc_features( resampled = resample_audios(audios, resample_rate=expected_sr) return cls.extract_sparc_features(resampled, lang=lang, device=device, resample=False) - # Load results - features_list = [] + # Load results. Values are tensors except ``ft_len``, which stays a + # plain scalar (a float on the error path, an int otherwise), so the + # element type is widened rather than forcing a tensor and changing + # what consumers read. + features_list: List[Dict[str, Any]] = [] for out_path in output.get("output_paths", []): with open(out_path) as f: raw = json.load(f) diff --git a/src/senselab/audio/tasks/forced_alignment/mms_fa.py b/src/senselab/audio/tasks/forced_alignment/mms_fa.py new file mode 100644 index 000000000..07906e4a0 --- /dev/null +++ b/src/senselab/audio/tasks/forced_alignment/mms_fa.py @@ -0,0 +1,84 @@ +"""Word-sequence forced alignment via torchaudio's MMS_FA bundle. + +Fills the previously-dead torchaudio slot of the forced-alignment task +(``constants.DEFAULT_ALIGN_MODELS_TORCH`` / the ``model_type == "torchaudio"`` +branch — see architecture-review.md F2/T047): a self-contained, lazily-imported +aligner that maps an ordered word list onto a 16 kHz waveform. Used by the +adaptive workflow's U3 consensus re-alignment; import by full module path +(``senselab.audio.tasks.forced_alignment.mms_fa``) — deliberately NOT re-exported +from the package ``__init__`` so importing it never pulls the transformers-based +aligner stack. + +The first call downloads the MMS_FA bundle (~1.2 GB); the SIGALRM timeout guard +keeps callers responsive on cold caches (fallback semantics are the caller's). +""" + +from __future__ import annotations + +import re +from typing import Any + +TARGET_SR = 16000 + + +def align_words_mms_fa( + wav_16k: Any, # noqa: ANN401 — 1-D float32 numpy array + words: list[str], + *, + timeout_s: float = 600.0, +) -> tuple[list[dict[str, float]] | None, str | None]: + """Align ``words`` (in order) to ``wav_16k`` → ``[{start, end}]`` matching 1:1. + + Returns ``(spans, None)`` on success or ``(None, reason)`` when torchaudio / + the bundle is unavailable, a word cannot be romanized for the MMS_FA + dictionary, the timeout fires (cold-cache bundle download), or the aligner + returns a span count mismatch. Never raises for these expected conditions. + """ + import signal + + try: + import torch # noqa: PLC0415 + from torchaudio.pipelines import MMS_FA as bundle # noqa: PLC0415, N811 + except ImportError as exc: + return None, f"aligner_backend_unavailable ({getattr(exc, 'name', exc)})" + + norm = [re.sub(r"[^a-z']", "", w.lower()) for w in words] + if any(not t for t in norm): + return None, "unalignable_tokens (non-romanizable words)" + + def _raise_timeout(signum: int, frame: Any) -> None: # noqa: ANN401, ARG001 + raise TimeoutError(f"mms_fa_timeout ({timeout_s}s)") + + old_handler = None + timer_armed = False + try: + old_handler = signal.signal(signal.SIGALRM, _raise_timeout) + signal.setitimer(signal.ITIMER_REAL, max(0.1, timeout_s)) + timer_armed = True + except ValueError: # not in main thread — proceed unguarded + old_handler = None + try: + model = bundle.get_model() + tokenizer = bundle.get_tokenizer() + aligner = bundle.get_aligner() + with torch.no_grad(): + emission, _ = model(torch.from_numpy(wav_16k).unsqueeze(0)) + spans = aligner(emission[0], tokenizer(norm)) + ratio = len(wav_16k) / emission.shape[1] + out = [] + for span in spans: + start = span[0].start * ratio / TARGET_SR + end = span[-1].end * ratio / TARGET_SR + out.append({"start": round(float(start), 4), "end": round(float(end), 4)}) + if len(out) != len(words): + return None, f"alignment_count_mismatch ({len(out)} != {len(words)})" + return out, None + except TimeoutError as exc: + return None, f"aligner_timeout ({exc})" + except Exception as exc: # noqa: BLE001 — expected-failure envelope for callers + return None, f"alignment_failed ({exc!r})" + finally: + if timer_armed: + signal.setitimer(signal.ITIMER_REAL, 0.0) + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) diff --git a/src/senselab/audio/tasks/quality_control/metrics.py b/src/senselab/audio/tasks/quality_control/metrics.py index 09310f752..f3e060297 100644 --- a/src/senselab/audio/tasks/quality_control/metrics.py +++ b/src/senselab/audio/tasks/quality_control/metrics.py @@ -128,9 +128,10 @@ def spectral_gating_snr_metric( Returns: float: Estimated segmental SNR in dB. """ - waveform: np.ndarray | torch.Tensor = audio.waveform - if isinstance(waveform, torch.Tensor): - waveform = waveform.numpy() + raw_waveform: np.ndarray | torch.Tensor = audio.waveform + # Bind the numpy view to its own name so the static type stays ndarray for + # librosa below, rather than remaining the input union. + waveform: np.ndarray = raw_waveform.numpy() if isinstance(raw_waveform, torch.Tensor) else raw_waveform if waveform.shape[0] > 1: waveform = np.mean(waveform, axis=0) @@ -200,7 +201,9 @@ def is_likely_clipped(channel: torch.Tensor, min_consecutive: int = 3) -> bool: return max_consecutive > min_consecutive for channel in waveform: - clipped_samples = 0 + # `.item()` on an integer count returns int|float to the type checker, so + # declare the accumulator wide enough for both. + clipped_samples: int | float = 0 max_val = torch.max(channel) # Check for clipping: either at/above threshold, or plateau pattern near threshold if max_val >= clip_threshold: @@ -224,14 +227,14 @@ def amplitude_modulation_depth_metric(audio: Audio) -> float: Returns: float: Amplitude modulation depth. """ - waveform = audio.waveform - if torch.numel(waveform) == 0: + raw_waveform = audio.waveform + if torch.numel(raw_waveform) == 0: return np.nan - assert waveform.ndim == 2, "Expected waveform shape (num_channels, num_samples)" + assert raw_waveform.ndim == 2, "Expected waveform shape (num_channels, num_samples)" - # Convert to numpy array if it's a torch tensor - if isinstance(waveform, torch.Tensor): - waveform = waveform.numpy() + # Convert to numpy array if it's a torch tensor (own name so the static type + # is ndarray for scipy below, not the Tensor it started as). + waveform: np.ndarray = raw_waveform.numpy() if isinstance(raw_waveform, torch.Tensor) else raw_waveform # Compute the analytic signal analytic_signal = scipy.signal.hilbert(waveform, axis=-1) @@ -423,11 +426,11 @@ def peak_snr_from_spectral_metric( Returns: float: Peak‑SNR in decibels. """ - waveform = audio.waveform - if torch.numel(waveform) == 0: + raw_waveform = audio.waveform + if torch.numel(raw_waveform) == 0: return np.nan - if isinstance(waveform, torch.Tensor): - waveform = waveform.numpy() + # Own name so the static type is ndarray for librosa below. + waveform: np.ndarray = raw_waveform.numpy() if isinstance(raw_waveform, torch.Tensor) else raw_waveform # Collapse to mono if multi-channel if waveform.ndim == 2 and waveform.shape[0] > 1: diff --git a/src/senselab/audio/tasks/scene_quality/__init__.py b/src/senselab/audio/tasks/scene_quality/__init__.py new file mode 100644 index 000000000..88b52e630 --- /dev/null +++ b/src/senselab/audio/tasks/scene_quality/__init__.py @@ -0,0 +1,12 @@ +"""Scene-quality models for the audio_analysis workflow. + +Currently wraps ``pyannote/brouhaha`` — a multitask model that predicts +per-frame voice activity, signal-to-noise ratio (SNR), and room-acoustics +clarity (C50) in a single forward pass. Used by the presence axis to source +the ``quality_snr`` / ``quality_reverb`` degradation scores and a second +frame-level speech-presence voter. +""" + +from senselab.audio.tasks.scene_quality.brouhaha import BrouhahaFrames, extract_brouhaha_frames + +__all__ = ["BrouhahaFrames", "extract_brouhaha_frames"] diff --git a/src/senselab/audio/tasks/scene_quality/brouhaha.py b/src/senselab/audio/tasks/scene_quality/brouhaha.py new file mode 100644 index 000000000..ecb237243 --- /dev/null +++ b/src/senselab/audio/tasks/scene_quality/brouhaha.py @@ -0,0 +1,281 @@ +"""``pyannote/brouhaha`` — joint per-frame VAD + SNR + C50 via an isolated venv. + +Brouhaha (Lavechin et al., 2022, arXiv:2210.13248) predicts, per frame and in a +single forward pass: + +- **VAD** — speech-presence probability in ``[0, 1]``; +- **SNR** — estimated signal-to-noise ratio in dB; +- **C50** — room-acoustics clarity in dB (higher = less reverberant). + +The scene-quality workflow uses the SNR/C50 heads for the ``quality_snr`` / +``quality_reverb`` degradation scores and the VAD head as a second frame-level +speech-presence voter. + +**Why a subprocess venv.** The ``pyannote/brouhaha`` checkpoint is not loadable +by our main environment: its custom multitask model class lives in the +GitHub-only ``brouhaha`` package (``brouhaha-vad``), which pins +``pyannote.audio>=3.1,<3.3.1``, ``speechbrain<1.0`` and ``numpy<2.0`` — all +incompatible with senselab's ``pyannote-audio>=4.0`` / ``speechbrain>=1.0``. +So Brouhaha runs in a dedicated venv (same pattern as the NeMo / Qwen ASR +backends), isolated from the main install. The model is gated on HuggingFace; +the worker reads ``HF_TOKEN`` from the environment (preserved across the +subprocess boundary). + +Long recordings are split into overlapping ~10 s chunks in the parent process +and stitched back into one continuous per-frame timeline via +``stitch_frames`` — flat memory, native ~17 ms resolution (shared with the +segmentation-3.0 extractor). +""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np + +from senselab.audio.data_structures import Audio +from senselab.audio.tasks.voice_activity_detection.frame_posteriors import stitch_frames +from senselab.utils.data_structures import DeviceType, _select_device_and_dtype +from senselab.utils.data_structures.logging import logger +from senselab.utils.subprocess_venv import _clean_subprocess_env, ensure_venv, parse_subprocess_result, venv_python + +BROUHAHA_MODEL_ID = "pyannote/brouhaha" +BROUHAHA_REVISION = "main" + +# Dedicated venv for the old brouhaha-vad stack. torch/torchaudio are routed +# through the CUDA-aware PyTorch index by ``ensure_venv`` (see CLAUDE.md); the +# pins target the pyannote-audio 3.x era brouhaha requires. numpy<2.0 and +# speechbrain<1.0 come transitively via brouhaha-vad's own requirements. +_BROUHAHA_VENV = "brouhaha" +_BROUHAHA_REQUIREMENTS = [ + "brouhaha @ git+https://github.com/marianne-m/brouhaha-vad.git", + "torch>=2.0,<2.3", + "torchaudio>=2.0,<2.3", + "numpy<2.0", + # brouhaha-vad pins pyannote 3.x but not huggingface_hub; the latest hub + # dropped the `use_auth_token` kwarg pyannote 3.x still passes to + # hf_hub_download. Pin to the era that keeps it. + "huggingface_hub>=0.19,<0.24", + "soundfile", +] +_BROUHAHA_PYTHON = "3.11" +# The ``torch>=2.0,<2.3`` pin above has no linux-x86_64 wheel on PyTorch's newer +# CUDA indexes (cu124/cu126/cu128), so on a modern-CUDA host the default +# host-keyed index selection makes the Stage-1 install unsatisfiable and the +# venv never builds. cu121 is the newest index that still publishes a +# ``torch<2.3`` wheel (``torch==2.2.2+cu121`` + matching torchaudio), and its +# wheels are forward-compatible with 12.x/13.x drivers for inference. Cap the +# index here so brouhaha builds on every GPU host. +_BROUHAHA_MAX_CUDA_VERSION = (12, 1) + +# Chunking for long recordings (mirrors the segmentation extractor's grid). +_CHUNK_S = 10.0 +_CHUNK_STEP_S = 8.0 + +# Multitask output channel order (frames, 3): [VAD, SNR dB, C50 dB]. +_VAD_CHANNEL = 0 +_SNR_CHANNEL = 1 +_C50_CHANNEL = 2 + +# Worker — runs inside the brouhaha venv. Loads the gated model (HF_TOKEN from +# env), runs whole-window inference per chunk wav, and saves each chunk's +# per-frame (VAD, SNR, C50) array as .npy for the parent to stitch. +_BROUHAHA_WORKER_SCRIPT = r""" +import json +import os +import sys + +try: + import numpy as np + import torch + from pyannote.audio import Inference, Model + + args = json.loads(sys.stdin.read()) + token = os.environ.get("HF_TOKEN") + dev = "cuda" if args["device"] == "cuda" and torch.cuda.is_available() else "cpu" + + model = Model.from_pretrained(args["model_name"], use_auth_token=token) + inference = Inference(model, window="whole", device=torch.device(dev)) + try: + hop = float(model.receptive_field.step) + except Exception: + hop = None + + results = [] + for ch in args["chunks"]: + out = inference(ch["path"]) + data = out.data if hasattr(out, "sliding_window") and hasattr(out, "data") else np.asarray(out) + data = np.asarray(data, dtype=np.float64) + if hop is None: + try: + hop = float(out.sliding_window.step) + except Exception: + hop = 0.01 + fname = "chunk_%d_%d.npy" % (ch["audio_idx"], int(round(ch["start_s"] * 1000))) + npy_path = os.path.join(args["out_dir"], fname) + np.save(npy_path, data) + results.append({"npy": npy_path, "start_s": ch["start_s"], "audio_idx": ch["audio_idx"], "hop": hop}) + + print(json.dumps({"results": results})) +except Exception as exc: + import traceback + + err = {"type": type(exc).__name__, "message": str(exc), "traceback": traceback.format_exc(limit=5)} + print(json.dumps({"error": err})) + sys.exit(1) +""" + + +@dataclass +class BrouhahaFrames: + """Per-frame Brouhaha outputs for one audio. + + Attributes: + vad: ``(num_frames,)`` speech-presence probability in ``[0, 1]``. + snr_db: ``(num_frames,)`` estimated SNR in dB. + c50_db: ``(num_frames,)`` estimated C50 (clarity) in dB. + frame_hop_s: seconds between consecutive frame starts. + """ + + vad: np.ndarray + snr_db: np.ndarray + c50_db: np.ndarray + frame_hop_s: float + + def mean_in_window(self, start_s: float, end_s: float) -> tuple[float, float, float]: + """Return ``(mean vad, mean snr_db, mean c50_db)`` over frames overlapping ``[start, end)``. + + Any component with no overlapping frames returns ``nan`` for that value. + """ + if self.frame_hop_s <= 0 or self.vad.size == 0: + return (float("nan"), float("nan"), float("nan")) + lo = max(0, int(np.floor(start_s / self.frame_hop_s))) + hi = min(self.vad.size, int(np.ceil(end_s / self.frame_hop_s))) + if hi <= lo: + return (float("nan"), float("nan"), float("nan")) + return ( + float(np.nanmean(self.vad[lo:hi])), + float(np.nanmean(self.snr_db[lo:hi])), + float(np.nanmean(self.c50_db[lo:hi])), + ) + + +def extract_brouhaha_frames( + audios: list[Audio], + device: Optional[DeviceType] = None, + model_id: str = BROUHAHA_MODEL_ID, + revision: str = BROUHAHA_REVISION, +) -> list[Optional[BrouhahaFrames]]: + """Run Brouhaha (in its isolated venv) once per audio, returning per-frame VAD/SNR/C50. + + Args: + audios: mono 16 kHz clips. + device: inference device (CPU or CUDA). + model_id: Brouhaha HF model id. + revision: model revision. + + Returns: + One ``BrouhahaFrames`` per input, or ``None`` for an audio the worker + failed on. If the venv cannot be built or the worker fails wholesale + (e.g. gated access not granted, install failure), every entry is + ``None`` — the workflow then emits null quality columns rather than + aborting (FR-023). The analyze_audio.py script treats scene quality as + required and fails loudly on all-null instead of silently degrading. + """ + if not audios: + return [] + + device_type, _ = _select_device_and_dtype( + user_preference=device, compatible_devices=[DeviceType.CUDA, DeviceType.CPU] + ) + + try: + venv_dir = ensure_venv( + _BROUHAHA_VENV, + _BROUHAHA_REQUIREMENTS, + python_version=_BROUHAHA_PYTHON, + max_cuda_version=_BROUHAHA_MAX_CUDA_VERSION, + ) + python = venv_python(venv_dir) + except Exception as exc: # noqa: BLE001 — venv build failure degrades to null (FR-023) + logger.warning(f"Failed to prepare Brouhaha venv: {exc}. Scene-quality signals will be null.") + return [None] * len(audios) + + with tempfile.TemporaryDirectory(prefix="senselab-brouhaha-") as tmpdir: + tmp = Path(tmpdir) + out_dir = tmp / "out" + out_dir.mkdir(parents=True, exist_ok=True) + + # Chunk each audio in the parent via senselab's sliding-window generator + # (overlapping ~10 s windows); the worker runs whole-window inference per + # chunk and the parent stitches. A clip <= one window yields a single chunk. + chunk_specs: list[dict] = [] + for ai, audio in enumerate(audios): + sr = int(audio.sampling_rate) + win_samples = int(_CHUNK_S * sr) + step_samples = max(1, int(_CHUNK_STEP_S * sr)) + for k, chunk in enumerate(audio.window_generator(win_samples, step_samples)): + start_s = (k * step_samples) / sr + path = str(tmp / f"audio_{ai}_{int(round(start_s * 1000))}.wav") + chunk.save_to_file(path) + chunk_specs.append({"path": path, "start_s": start_s, "audio_idx": ai}) + + input_json = json.dumps( + { + "chunks": chunk_specs, + "model_name": model_id, + "revision": revision, + "device": device_type.value, + "out_dir": str(out_dir), + } + ) + + try: + result = subprocess.run( + [python, "-c", _BROUHAHA_WORKER_SCRIPT], + input=input_json, + capture_output=True, + text=True, + timeout=1800, + env=_clean_subprocess_env(), + ) + output = parse_subprocess_result(result, "Brouhaha") + except Exception as exc: # noqa: BLE001 — worker failure degrades to null (FR-023) + logger.warning(f"Brouhaha worker failed: {exc}. Scene-quality signals will be null.") + return [None] * len(audios) + + # Regroup per-chunk results by audio index, load npy, and stitch. + by_audio: dict[int, list[dict]] = {} + for entry in output.get("results", []): + by_audio.setdefault(int(entry["audio_idx"]), []).append(entry) + + results: list[Optional[BrouhahaFrames]] = [] + for ai in range(len(audios)): + entries = by_audio.get(ai) + if not entries: + results.append(None) + continue + try: + arrays = [np.load(e["npy"]) for e in entries] + starts = [float(e["start_s"]) for e in entries] + hop = float(entries[0].get("hop") or 0.0) + data = stitch_frames(arrays, starts, hop) if hop > 0 else np.zeros((0, 3)) + if data.ndim != 2 or data.shape[1] <= _C50_CHANNEL: + raise ValueError(f"unexpected Brouhaha output shape {data.shape}") + results.append( + BrouhahaFrames( + vad=data[:, _VAD_CHANNEL], + snr_db=data[:, _SNR_CHANNEL], + c50_db=data[:, _C50_CHANNEL], + frame_hop_s=hop, + ) + ) + except (OSError, ValueError, KeyError) as exc: + logger.warning(f"Failed to assemble Brouhaha frames for audio {ai}: {exc}") + results.append(None) + return results diff --git a/src/senselab/audio/tasks/speaker_diarization/nvidia.py b/src/senselab/audio/tasks/speaker_diarization/nvidia.py index a443312a7..288d62135 100644 --- a/src/senselab/audio/tasks/speaker_diarization/nvidia.py +++ b/src/senselab/audio/tasks/speaker_diarization/nvidia.py @@ -25,6 +25,9 @@ "pyarrow<18", # pyarrow 24+ removed PyExtensionType "matplotlib", "soundfile", + # NeMo pulls librosa without a floor; pin numba so uv doesn't backtrack + # librosa -> numba -> llvmlite 0.36.0 (no Python 3.12 support). See qwen.py. + "numba>=0.60", ] # NOTE: The `lightning` package was removed from PyPI (April 2026). # NeMo requires it but pytorch-lightning is not a drop-in replacement. diff --git a/src/senselab/audio/tasks/speech_to_text/api.py b/src/senselab/audio/tasks/speech_to_text/api.py index fcc2e90f3..509983749 100644 --- a/src/senselab/audio/tasks/speech_to_text/api.py +++ b/src/senselab/audio/tasks/speech_to_text/api.py @@ -9,6 +9,7 @@ from senselab.audio.data_structures import Audio from senselab.audio.tasks.speech_to_text.canary_qwen import CanaryQwenASR +from senselab.audio.tasks.speech_to_text.crisperwhisper import CrisperWhisperASR from senselab.audio.tasks.speech_to_text.granite import GraniteSpeechASR from senselab.audio.tasks.speech_to_text.huggingface import HuggingFaceASR from senselab.audio.tasks.speech_to_text.nemo import NeMoASR @@ -30,6 +31,12 @@ # Qwen3-ForcedAligner companion for word-level timestamps. _QWEN_ASR_PREFIXES = ("Qwen/Qwen3-ASR",) +# CrisperWhisper 2.0 prefix — route to a separate CT2 subprocess venv +# (crisperwhisper.py) using the `crisperwhisper` package. Verbatim transcription +# with native word timestamps + per-word confidence. (v1 "nyralabs/CrisperWhisper" +# is plain transformers and still goes through the HuggingFaceASR path.) +_CRISPER_PREFIXES = ("nyralabs/CrisperWhisper2.0",) + # IBM Granite Speech prefix — route to a separate granite subprocess venv # (granite.py). Granite uses GraniteSpeechProcessor's (text, audio, device) # signature, not the standard HF ASR pipeline contract, so it cannot be @@ -134,6 +141,13 @@ def transcribe_audios( if "forced_aligner" in kwargs: qwen_kwargs["forced_aligner"] = kwargs.pop("forced_aligner") return QwenASR.transcribe_with_qwen(audios=audios, model=model, device=device, **qwen_kwargs) + elif isinstance(model, HFModel) and str(model.path_or_uri).startswith(_CRISPER_PREFIXES): + crisper_kwargs: dict[str, Any] = {} + if "language" in kwargs: + crisper_kwargs["language"] = kwargs.pop("language") + return CrisperWhisperASR.transcribe_with_crisperwhisper( + audios=audios, model=model, device=device, **crisper_kwargs + ) elif isinstance(model, HFModel) and str(model.path_or_uri).startswith(_GRANITE_PREFIXES): return GraniteSpeechASR.transcribe_with_granite(audios=audios, model=model, device=device) elif isinstance(model, HFModel): diff --git a/src/senselab/audio/tasks/speech_to_text/canary_qwen.py b/src/senselab/audio/tasks/speech_to_text/canary_qwen.py index 02f048acd..53675b242 100644 --- a/src/senselab/audio/tasks/speech_to_text/canary_qwen.py +++ b/src/senselab/audio/tasks/speech_to_text/canary_qwen.py @@ -53,6 +53,16 @@ "pyarrow<18", "matplotlib", "soundfile", + # NeMo pulls librosa without a floor; on Python 3.12 uv otherwise backtracks + # librosa -> numba -> llvmlite to llvmlite 0.36.0 (no 3.12 support). Pin a + # modern numba floor so the chain resolves to llvmlite>=0.43 (same fix as the + # qwen-asr venv). + "numba>=0.60", + # SALM's Qwen LM uses a LoRA adapter -> the worker imports peft at load time, + # but NeMo doesn't always pull it transitively on a fresh resolve. Pin it + # explicitly (the senselab core lists peft for Granite; the isolated venv needs + # its own copy). Surfaced as "No module named 'peft'" at canary load. + "peft>=0.13", ] _CANARY_PYTHON = "3.12" diff --git a/src/senselab/audio/tasks/speech_to_text/crisperwhisper.py b/src/senselab/audio/tasks/speech_to_text/crisperwhisper.py new file mode 100644 index 000000000..86e0b4161 --- /dev/null +++ b/src/senselab/audio/tasks/speech_to_text/crisperwhisper.py @@ -0,0 +1,215 @@ +"""CrisperWhisper 2.0 (verbatim, word-timed) via an isolated subprocess venv. + +CrisperWhisper 2.0 (nyralabs) is a Whisper-derived model tuned for **verbatim** +transcription and **word-level timestamps** (~30-40 ms boundary error). It ships +as the ``crisperwhisper`` pip package (2.x) with a CTranslate2 backend +(``crisperwhisper[ct2]``) rather than plain ``transformers`` weights, so it runs +in its own venv (same pattern as the Qwen / Canary / Brouhaha backends) — the +CT2 fork (``ctranslate2-crisperwhisper``) must not leak into the senselab core. + +The worker returns per-word timestamps and native per-word confidence (when the +library exposes it) so the utterance axis can consume a native uncertainty +signal via ``ScriptLine.score`` (line-level) and each word chunk's ``score``. +""" + +from __future__ import annotations + +import json +import platform +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import List, Optional + +from senselab.audio.data_structures import Audio +from senselab.utils.data_structures import DeviceType, HFModel, ScriptLine, _select_device_and_dtype +from senselab.utils.subprocess_venv import _clean_subprocess_env, ensure_venv, parse_subprocess_result, venv_python + +_CRISPER_VENV = "crisperwhisper" +# Backend is platform-selected. ``ctranslate2-crisperwhisper`` only publishes +# Linux x86_64 wheels, so the fast CT2 path is used there (the GPU / CI target); +# everywhere else (e.g. macOS arm64 dev) falls back to the transformers backend, +# which loads the same model's safetensors weights. Both extras expose the same +# ``CrisperWhisperModel`` API, so the worker is backend-agnostic. +_IS_LINUX_X86 = sys.platform.startswith("linux") and platform.machine().lower() in ("x86_64", "amd64") +if _IS_LINUX_X86: + _CRISPER_REQUIREMENTS = ["crisperwhisper[ct2]==2.0.1"] +else: + _CRISPER_REQUIREMENTS = [ + "crisperwhisper[transformers]==2.0.1", + # The library imports `ctranslate2` at module top (engine/hallucination) + # even on the transformers path, but the [transformers] extra doesn't + # install it and the CT2 *fork* is Linux-x86-only. Standard ctranslate2 + # has macOS-arm64 wheels and satisfies those imports (the transformers + # backend doesn't actually run CT2 inference). + "ctranslate2>=4.0", + # Pin torch/torchaudio explicitly so ensure_venv routes them through the + # CUDA-aware PyTorch index (the [transformers] extra pulls torch>=2.4). + "torch>=2.4", + "torchaudio>=2.4", + ] +_CRISPER_PYTHON = "3.12" + +# Backend token passed to CrisperWhisperModel(..., backend=...). "auto" would try +# ct2 first (and fail off Linux x86_64), so we pin it explicitly per platform. +_CRISPER_BACKEND = "ct2" if _IS_LINUX_X86 else "transformers" + + +# Worker — runs inside the isolated venv. +_CRISPER_WORKER_SCRIPT = r""" +import json +import sys + +try: + from crisperwhisper import CrisperWhisperModel + + args = json.loads(sys.stdin.read()) + audio_paths = args["audio_paths"] + model_id = args["model_id"] + backend = args.get("backend", "auto") + device = args.get("device", "auto") + compute_type = args.get("compute_type", "float32") + language = args.get("language") or "en" + + model = CrisperWhisperModel(model_id, backend=backend, device=device, compute_type=compute_type) + + def _first_attr(obj, names): + for n in names: + v = getattr(obj, n, None) + if v is not None: + return v + return None + + results = [] + for path in audio_paths: + r = model.transcribe(path, language=language, word_timestamps=True) + words = [] + for w in (getattr(r, "words", None) or []): + conf = _first_attr(w, ("probability", "confidence", "score", "prob")) + words.append({ + "text": _first_attr(w, ("word", "text")) or "", + "start": float(w.start), + "end": float(w.end), + "score": (float(conf) if conf is not None else None), + }) + line_conf = _first_attr(r, ("confidence", "avg_logprob", "score")) + if line_conf is None: + cs = [w["score"] for w in words if w["score"] is not None] + line_conf = (sum(cs) / len(cs)) if cs else None + results.append({ + "text": getattr(r, "text", "") or "", + "language": getattr(r, "language", language), + "words": words, + "score": (float(line_conf) if line_conf is not None else None), + }) + + print(json.dumps({"results": results})) +except Exception as exc: + import traceback + err = {"type": type(exc).__name__, "message": str(exc), "traceback": traceback.format_exc(limit=5)} + print(json.dumps({"error": err})) + sys.exit(1) +""" + + +class CrisperWhisperASR: + """CrisperWhisper 2.0 transcription via its isolated CT2 subprocess venv. + + Routed automatically by ``speech_to_text.api`` when the model id matches the + ``nyralabs/CrisperWhisper2.0`` prefix. Returns one ``ScriptLine`` per audio + with verbatim ``text``, per-word ``chunks`` (with ``score`` = native word + confidence when available), and a line-level ``score``. + """ + + @classmethod + def transcribe_with_crisperwhisper( + cls, + audios: List[Audio], + model: Optional[HFModel] = None, + device: Optional[DeviceType] = None, + language: Optional[str] = None, + ) -> List[ScriptLine]: + """Transcribe audios with CrisperWhisper 2.0 via the dedicated subprocess venv. + + Args: + audios: Audio clips (mono, 16 kHz expected). + model: HF model id (default ``nyralabs/CrisperWhisper2.0_turbo``). + device: CPU or CUDA (CT2 auto-uses the GPU when available). + language: Force a language (default ``en``); CrisperWhisper is en/de. + + Returns: + One ``ScriptLine`` per input with verbatim ``text``, word-level + ``chunks`` carrying timestamps + ``score`` (native word confidence + when exposed), and a line-level ``score``. + """ + model_id = str(model.path_or_uri) if model is not None else "nyralabs/CrisperWhisper2.0_turbo" + device_type, _ = _select_device_and_dtype( + user_preference=device, compatible_devices=[DeviceType.CUDA, DeviceType.CPU] + ) + # float16 only on CUDA; CPU (e.g. macOS transformers backend) needs float32. + device_str = "cuda" if device_type == DeviceType.CUDA else "cpu" + compute_type = "float16" if device_str == "cuda" else "float32" + + venv_dir = ensure_venv(_CRISPER_VENV, _CRISPER_REQUIREMENTS, python_version=_CRISPER_PYTHON) + python = venv_python(venv_dir) + + with tempfile.TemporaryDirectory(prefix="senselab-crisperwhisper-") as tmpdir: + tmp = Path(tmpdir) + audio_paths: List[str] = [] + for i, audio in enumerate(audios): + path = str(tmp / f"audio_{i}.wav") + audio.save_to_file(path) + audio_paths.append(path) + + input_json = json.dumps( + { + "audio_paths": audio_paths, + "model_id": model_id, + "backend": _CRISPER_BACKEND, + "device": device_str, + "compute_type": compute_type, + "language": language or "en", + } + ) + result = subprocess.run( + [python, "-c", _CRISPER_WORKER_SCRIPT], + input=input_json, + capture_output=True, + text=True, + timeout=1800, + env=_clean_subprocess_env(), + ) + output = parse_subprocess_result(result, "CrisperWhisper 2.0") + + results: List[ScriptLine] = [] + for entry in output.get("results", []): + words = entry.get("words") or [] + chunks: Optional[List[ScriptLine]] = None + line_start: Optional[float] = None + line_end: Optional[float] = None + if words: + chunks = [ + ScriptLine( + text=w["text"], + start=float(w["start"]), + end=float(w["end"]), + score=(float(w["score"]) if w.get("score") is not None else None), + ) + for w in words + ] + chunks.sort(key=lambda c: c.start if c.start is not None else 0.0) + starts = [c.start for c in chunks if c.start is not None] + ends = [c.end for c in chunks if c.end is not None] + line_start = min(starts) if starts else None + line_end = max(ends) if ends else None + results.append( + ScriptLine( + text=entry.get("text", ""), + start=line_start, + end=line_end, + chunks=chunks, + score=(float(entry["score"]) if entry.get("score") is not None else None), + ) + ) + return results diff --git a/src/senselab/audio/tasks/speech_to_text/huggingface.py b/src/senselab/audio/tasks/speech_to_text/huggingface.py index e18d413b6..150aaf68a 100644 --- a/src/senselab/audio/tasks/speech_to_text/huggingface.py +++ b/src/senselab/audio/tasks/speech_to_text/huggingface.py @@ -11,6 +11,11 @@ from transformers import Pipeline, pipeline from senselab.audio.data_structures import Audio +from senselab.audio.tasks.speech_to_text.token_confidence import ( + capture_token_confidence, + merge_confidence_blocks, + whisper_token_ids, +) from senselab.utils.data_structures import ( DeviceType, HFModel, @@ -22,6 +27,44 @@ from senselab.utils.data_structures.model import get_huggingface_token +def _attach_token_confidence(script_lines: List[ScriptLine], confidences: List[Dict[str, Any]]) -> None: + """Attach captured per-token confidence onto the emitted `ScriptLine`s, in place. + + Alignment rules, in order: + + 1. **One block per line** — the batch case; assign 1:1. + 2. **Single line, many blocks** — Whisper long-form calls ``generate`` once per + 30 s window, so one transcript accumulates several blocks. Merge them: + concatenate the entropies, average the log-probabilities, and keep the + maximum no-speech probability (the most silence-like window dominates). + 3. **Anything else** — counts we can't map confidently; leave the fields + ``None`` rather than risk attributing one audio's confidence to another. + """ + if not confidences: + return + + if len(confidences) == len(script_lines): + for line, block in zip(script_lines, confidences): + _set_confidence(line, block) + return + + if len(script_lines) == 1: + _set_confidence(script_lines[0], merge_confidence_blocks(confidences)) + return + + logger.debug( + f"Token confidence not attached: {len(confidences)} generate call(s) for " + f"{len(script_lines)} transcript(s) — no unambiguous mapping." + ) + + +def _set_confidence(line: ScriptLine, block: Dict[str, Any]) -> None: + """Copy one confidence block onto a `ScriptLine`.""" + line.avg_logprob = block.get("avg_logprob") + line.no_speech_prob = block.get("no_speech_prob") + line.token_entropy = block.get("token_entropy") + + class HuggingFaceASR: """Factory for creating and caching Hugging Face ASR pipelines. @@ -262,11 +305,22 @@ def _sanitize_timestamps_recursive(obj: Union[Dict, List, object]) -> Union[Dict # Take the start time of the transcription start_time_transcription = time.time() - # Run the pipeline - transcriptions = pipe( - formatted_audios, - generate_kwargs={"language": f"{language.name.lower()}", "num_beams": 1} if language else {"num_beams": 1}, - ) + # Run the pipeline, observing the decoder's own per-token confidence as it + # goes (FR-017). The pipeline discards generate's scores before postprocess, + # so the seam wraps `model.generate` rather than passing `output_scores` + # through `generate_kwargs` — see token_confidence.py for why. + no_speech_token_id, special_token_ids = whisper_token_ids(pipe) + with capture_token_confidence( + pipe, + no_speech_token_id=no_speech_token_id, + special_token_ids=special_token_ids, + ) as token_confidences: + transcriptions = pipe( + formatted_audios, + generate_kwargs=( + {"language": f"{language.name.lower()}", "num_beams": 1} if language else {"num_beams": 1} + ), + ) # Take the end time of the transcription end_time_transcription = time.time() @@ -281,4 +335,6 @@ def _sanitize_timestamps_recursive(obj: Union[Dict, List, object]) -> Union[Dict transcriptions = _sanitize_timestamps_recursive(transcriptions) # Convert the pipeline output to ScriptLine objects - return [ScriptLine.from_dict(cast(Dict[str, Any], t)) for t in cast(List, transcriptions)] + script_lines = [ScriptLine.from_dict(cast(Dict[str, Any], t)) for t in cast(List, transcriptions)] + _attach_token_confidence(script_lines, token_confidences) + return script_lines diff --git a/src/senselab/audio/tasks/speech_to_text/nemo.py b/src/senselab/audio/tasks/speech_to_text/nemo.py index d0e235030..4b49d82ea 100644 --- a/src/senselab/audio/tasks/speech_to_text/nemo.py +++ b/src/senselab/audio/tasks/speech_to_text/nemo.py @@ -32,6 +32,9 @@ "pyarrow<18", # pyarrow 24+ removed PyExtensionType "matplotlib", "soundfile", + # NeMo pulls librosa without a floor; pin numba so uv doesn't backtrack + # librosa -> numba -> llvmlite 0.36.0 (no Python 3.12 support). See qwen.py. + "numba>=0.60", ] # NOTE: Same `lightning` package issue as diarization — see nvidia.py comment. _NEMO_PYTHON = "3.12" diff --git a/src/senselab/audio/tasks/speech_to_text/qwen.py b/src/senselab/audio/tasks/speech_to_text/qwen.py index 3018ae85d..ce94b4900 100644 --- a/src/senselab/audio/tasks/speech_to_text/qwen.py +++ b/src/senselab/audio/tasks/speech_to_text/qwen.py @@ -19,7 +19,7 @@ import subprocess import tempfile from pathlib import Path -from typing import List, Optional +from typing import Any, List, Optional, Sequence, Tuple from senselab.audio.data_structures import Audio from senselab.utils.data_structures import DeviceType, HFModel, ScriptLine, _select_device_and_dtype @@ -40,6 +40,11 @@ # exists to prevent). "torch>=2.8,<2.9", "torchaudio>=2.8,<2.9", + # qwen-asr depends on librosa without a floor; on Python 3.12 uv otherwise + # backtracks librosa -> numba -> llvmlite to the ancient llvmlite 0.36.0, + # which has no 3.12 support ("Cannot install on Python version 3.12.0"). + # Pin a modern numba floor so the chain resolves to llvmlite>=0.43 (3.12-ok). + "numba>=0.60", ] _QWEN_PYTHON = "3.12" @@ -48,6 +53,51 @@ # (Qwen3-ASR-1.7B / Qwen3-ASR-3B / etc.). _DEFAULT_FORCED_ALIGNER = "Qwen/Qwen3-ForcedAligner-0.6B" +# Standalone forced-alignment worker — aligns an existing (text, audio) pair to +# per-word timestamps via Qwen3ForcedAligner. Used to align text-only ASR output +# (e.g. Canary) that carries no native timestamps. Qwen3ForcedAligner.align() +# takes batched (audio, text, language) lists and returns one iterable +# ForcedAlignResult per input, each yielding spans with .text/.start_time/.end_time. +_QWEN_ALIGN_WORKER_SCRIPT = r""" +import json +import sys + +try: + import torch + from qwen_asr import Qwen3ForcedAligner + + args = json.loads(sys.stdin.read()) + pairs = args["pairs"] + aligner_name = args["aligner_model"] + device = args["device"] + + fa = Qwen3ForcedAligner.from_pretrained(aligner_name) + if device == "cuda" and torch.cuda.is_available(): + try: + fa.model = fa.model.cuda() + except Exception as cuda_exc: + print(f"WARN: Qwen aligner CUDA placement failed ({cuda_exc!r}); using CPU.", file=sys.stderr) + + audios = [p["audio_path"] for p in pairs] + texts = [p["text"] for p in pairs] + langs = [p.get("language") or "en" for p in pairs] + aligned = fa.align(audios, texts, langs) + + results = [] + for r in aligned: + chunks = [] + for span in r: + chunks.append({"text": span.text, "start": float(span.start_time), "end": float(span.end_time)}) + results.append({"chunks": chunks}) + + print(json.dumps({"results": results})) +except Exception as exc: + import traceback + err = {"type": type(exc).__name__, "message": str(exc), "traceback": traceback.format_exc(limit=5)} + print(json.dumps({"error": err})) + sys.exit(1) +""" + # Worker script — runs inside the isolated venv. # Uses Qwen3ASRModel.from_pretrained's built-in `forced_aligner` kwarg # (a string id) so the wrapper handles aligner construction internally. @@ -215,7 +265,7 @@ def transcribe_with_qwen( # Sort by start so out-of-order aligner output doesn't yield a # negative line span; pick the min start and max end across all # chunks rather than trusting position-0/-1. - chunks.sort(key=lambda c: (c.start if c.start is not None else 0.0)) + chunks.sort(key=lambda c: c.start if c.start is not None else 0.0) valid_starts = [c.start for c in chunks if c.start is not None] valid_ends = [c.end for c in chunks if c.end is not None] line_start = min(valid_starts) if valid_starts else None @@ -230,3 +280,83 @@ def transcribe_with_qwen( ) return results + + @classmethod + def align_with_qwen( + cls, + data: Sequence[Tuple[Audio, ScriptLine, Any]], + levels_to_keep: Optional[dict] = None, # noqa: ARG003 — accepted for align_transcriptions parity + aligner_model: Optional[str] = None, + device: Optional[DeviceType] = None, + **_kwargs: Any, # noqa: ANN401 — parity with align_transcriptions' extra kwargs + ) -> List[List[ScriptLine]]: + """Force-align existing (audio, transcript) pairs with Qwen3-ForcedAligner. + + Drop-in alternative to ``forced_alignment.align_transcriptions`` for + text-only ASR output that lacks native timestamps (e.g. Canary). Runs in + the shared ``qwen-asr`` subprocess venv. + + Args: + data: ``(audio, ScriptLine(text=...), Language|None)`` tuples — same + call form the script uses for ``align_transcriptions``. + levels_to_keep: Ignored (Qwen emits word-level spans); accepted only + for signature parity so the caller can swap aligners freely. + aligner_model: Aligner id (default ``Qwen/Qwen3-ForcedAligner-0.6B``). + device: CPU or CUDA. + + Returns: + ``List[List[ScriptLine]]`` mirroring ``align_transcriptions``: one + inner list per input audio, holding a single utterance ``ScriptLine`` + whose ``chunks`` are the aligned word spans. + """ + aligner_name = aligner_model or _DEFAULT_FORCED_ALIGNER + device_type = device or _select_device_and_dtype(compatible_devices=[DeviceType.CUDA, DeviceType.CPU])[0] + + venv_dir = ensure_venv(_QWEN_VENV, _QWEN_REQUIREMENTS, python_version=_QWEN_PYTHON) + python = venv_python(venv_dir) + + with tempfile.TemporaryDirectory(prefix="senselab-qwen-align-") as tmpdir: + tmp = Path(tmpdir) + pairs: List[dict] = [] + for i, (audio, script, language) in enumerate(data): + path = str(tmp / f"audio_{i}.wav") + audio.save_to_file(path) + pairs.append( + { + "audio_path": path, + "text": script.text or "", + "language": getattr(language, "language_code", None) or "en", + } + ) + + input_json = json.dumps({"pairs": pairs, "aligner_model": aligner_name, "device": device_type.value}) + result = subprocess.run( + [python, "-c", _QWEN_ALIGN_WORKER_SCRIPT], + input=input_json, + capture_output=True, + text=True, + timeout=1800, + env=_clean_subprocess_env(), + ) + output = parse_subprocess_result(result, "Qwen forced aligner") + + aligned: List[List[ScriptLine]] = [] + for entry in output.get("results", []): + chunks = [ + ScriptLine(text=c["text"], start=float(c["start"]), end=float(c["end"])) + for c in (entry.get("chunks") or []) + ] + chunks.sort(key=lambda c: c.start if c.start is not None else 0.0) + if not chunks: + aligned.append([]) + continue + starts = [c.start for c in chunks if c.start is not None] + ends = [c.end for c in chunks if c.end is not None] + utterance = ScriptLine( + text=" ".join(c.text or "" for c in chunks).strip(), + start=min(starts) if starts else None, + end=max(ends) if ends else None, + chunks=chunks, + ) + aligned.append([utterance]) + return aligned diff --git a/src/senselab/audio/tasks/speech_to_text/token_confidence.py b/src/senselab/audio/tasks/speech_to_text/token_confidence.py new file mode 100644 index 000000000..331389958 --- /dev/null +++ b/src/senselab/audio/tasks/speech_to_text/token_confidence.py @@ -0,0 +1,337 @@ +"""Per-token ASR confidence extraction (feature 20260722-175022, FR-017). + +Whisper's decoder knows more about its own uncertainty than the transcript text +reveals: the per-step distribution over the vocabulary is peaked when the model is +sure and flat when it is guessing. That signal is what the utterance axis needs — +pairwise transcript disagreement only fires when two backends actually differ, +while token entropy registers a single model's private doubt. + +Why a capture seam rather than ``generate_kwargs``: + Transformers' ``AutomaticSpeechRecognitionPipeline._forward`` keeps only + ``sequences`` and ``token_timestamps`` from the generate output and drops the + scores/logits before ``postprocess`` ever sees them. Passing + ``output_scores=True`` through ``pipe(...)`` therefore has no observable + effect. :func:`capture_token_confidence` instead wraps ``pipe.model.generate`` + for the duration of one call, reads the logits off the returned + ``ModelOutput``, and restores the original method afterwards. The pipeline's + own behavior is untouched — we only observe. + +We request ``output_logits`` (raw, pre-processor) rather than ``output_scores`` +(post-processor) on purpose: Whisper's logits processors suppress +``<|nospeech|>`` to ``-inf``, which would drive the no-speech probability to a +constant 0, and they distort the entropy of the remaining distribution. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator, Sequence +from typing import Any, Optional + +import torch + +__all__ = [ + "capture_token_confidence", + "merge_confidence_blocks", + "token_confidence_from_logits", + "whisper_token_ids", +] + +_NO_SPEECH_TOKEN_SPELLINGS = ("<|nospeech|>", "<|nocaptions|>") +"""Vocabulary spellings of Whisper's no-speech marker, in preference order. + +The spelling is not stable across releases: ``openai/whisper-tiny`` ships +``<|nocaptions|>`` (id 50362) and maps ``<|nospeech|>`` to the *unk* id, while other +checkpoints do the reverse. Both are tried, and any hit equal to ``unk_token_id`` is +rejected — otherwise we'd read the probability of an unrelated token. +""" + + +def whisper_token_ids(pipe: Any) -> tuple[Optional[int], Optional[set]]: # noqa: ANN401 — a transformers pipeline + """Resolve ``(no_speech_token_id, special_token_ids)`` for a pipeline's tokenizer. + + Returns ``(None, None)`` for backends without a tokenizer (e.g. raw CTC models), + which is what makes the confidence fields degrade gracefully (FR-017). + + Args: + pipe: A transformers pipeline, or any object exposing ``tokenizer`` and + ``model.generation_config``. + + Returns: + The ``<|nospeech|>`` vocabulary id (or ``None`` if this backend has none) and + the set of special-token ids to exclude from ``avg_logprob`` (or ``None``). + """ + tokenizer = getattr(pipe, "tokenizer", None) + if tokenizer is None: + return None, None + + no_speech_id: Optional[int] = None + generation_config = getattr(getattr(pipe, "model", None), "generation_config", None) + for attr in ("no_speech_token_id", "no_speech_token"): + candidate = getattr(generation_config, attr, None) + if isinstance(candidate, int): + no_speech_id = candidate + break + + if no_speech_id is None: + unk = getattr(tokenizer, "unk_token_id", None) + for spelling in _NO_SPEECH_TOKEN_SPELLINGS: + try: + candidate = tokenizer.convert_tokens_to_ids(spelling) + except (AttributeError, KeyError, TypeError, ValueError): + continue + if isinstance(candidate, int) and candidate != unk: + no_speech_id = candidate + break + + # Guard the conversion: a stand-in/Mock tokenizer returns a non-iterable + # attribute here, and blowing up would fail the whole transcription over a + # purely additive signal. + special_ids: Optional[set] = None + raw_special = getattr(tokenizer, "all_special_ids", None) + if raw_special is not None: + try: + candidates = {int(i) for i in raw_special if isinstance(i, int)} + except TypeError: + candidates = set() + special_ids = candidates or None + return no_speech_id, special_ids + + +def merge_confidence_blocks(blocks: Sequence[dict[str, Any]]) -> dict[str, Any]: + """Fold several confidence blocks for the *same* transcript into one. + + Used for Whisper long-form, where one transcript is produced by several + ``generate`` calls (one per 30 s window) and each window yields its own block. + Entropies concatenate (they're a per-token sequence), log-probabilities average, + and the no-speech probability takes the maximum so the most silence-like window + dominates rather than being averaged away. + + Args: + blocks: Confidence blocks as returned by :func:`token_confidence_from_logits`. + + Returns: + A single merged block with the same three keys. + """ + entropies: list[float] = [] + logprobs: list[float] = [] + no_speech: list[float] = [] + for block in blocks: + raw_entropy = block.get("token_entropy") + if isinstance(raw_entropy, (list, tuple)): + entropies.extend(float(e) for e in raw_entropy) + elif raw_entropy is not None: + entropies.append(float(raw_entropy)) + if block.get("avg_logprob") is not None: + logprobs.append(float(block["avg_logprob"])) + if block.get("no_speech_prob") is not None: + no_speech.append(float(block["no_speech_prob"])) + return { + "token_entropy": entropies or None, + "avg_logprob": (sum(logprobs) / len(logprobs)) if logprobs else None, + "no_speech_prob": max(no_speech) if no_speech else None, + } + + +def token_confidence_from_logits( + *, + logits: Sequence[torch.Tensor], + sequences: torch.Tensor, + no_speech_token_id: Optional[int] = None, + special_token_ids: Optional[set[int]] = None, +) -> list[dict[str, Any]]: + """Derive per-token entropy, ``avg_logprob`` and ``no_speech_prob`` from logits. + + Args: + logits: One tensor per decoding step, each shaped ``(batch, vocab)``, as + returned by ``generate(..., output_logits=True)``. Raw (unprocessed) + logits are expected. + sequences: Generated token ids, shaped ``(batch, seq_len)``. For + encoder-decoder models ``seq_len`` exceeds ``len(logits)`` because it + carries the forced decoder prefix; the trailing ``len(logits)`` ids are + the scored ones, matching HF's transition-score alignment. + no_speech_token_id: Vocabulary id of Whisper's ``<|nospeech|>`` token. When + given, ``no_speech_prob`` is read from the first step's distribution — + Whisper's own definition. ``None`` for backends without the token. + special_token_ids: Ids excluded from ``avg_logprob`` (forced language / + task / timestamp markers). Entropy is still reported for every step. + + Returns: + One dict per batch row with keys ``token_entropy`` (list of nats, or + ``None``), ``avg_logprob`` (float or ``None``) and ``no_speech_prob`` + (float or ``None``). + + Example: + >>> import torch + >>> flat = torch.zeros(1, 4) # uniform over 4 tokens + >>> out = token_confidence_from_logits(logits=[flat], sequences=torch.tensor([[2]])) + >>> round(out[0]["token_entropy"][0], 4) == round(float(torch.log(torch.tensor(4.0))), 4) + True + """ + # Whisper's long-form path hands back unbatched per-segment tensors — + # logits ``(vocab,)`` and sequences ``(seq_len,)``. Normalize to the batched + # shapes so one code path serves both. Verified against transformers 5.5.4. + if sequences.ndim == 1: + sequences = sequences.unsqueeze(0) + logits = [step.unsqueeze(0) if step.ndim == 1 else step for step in logits] + + batch_size = int(sequences.shape[0]) if sequences.ndim >= 1 else 1 + steps = len(logits) + if steps == 0: + return [{"token_entropy": None, "avg_logprob": None, "no_speech_prob": None} for _ in range(batch_size)] + + excluded = special_token_ids or set() + + # Whisper reads the no-speech probability off the first generated position. + no_speech: list[Optional[float]] = [None] * batch_size + if no_speech_token_id is not None: + first = logits[0].detach().float() + if first.ndim == 2 and 0 <= no_speech_token_id < first.shape[-1]: + probs = torch.softmax(first, dim=-1)[:, no_speech_token_id] + no_speech = [float(p) for p in probs[:batch_size]] + + # Align the scored ids to the recorded steps (drop any forced prefix). + scored_ids = sequences[:, -steps:] if sequences.ndim == 2 and sequences.shape[1] >= steps else None + + entropies: list[list[float]] = [[] for _ in range(batch_size)] + chosen_logprobs: list[list[float]] = [[] for _ in range(batch_size)] + + for step, step_logits in enumerate(logits): + lg = step_logits.detach().float() + if lg.ndim != 2: + continue + logprobs = torch.log_softmax(lg, dim=-1) + # -Σ p·log p, computed from logprobs to avoid a second softmax. + step_entropy = -(logprobs.exp() * logprobs).sum(dim=-1) + for row in range(min(batch_size, lg.shape[0])): + entropies[row].append(float(step_entropy[row])) + if scored_ids is None: + continue + token_id = int(scored_ids[row, step]) + if token_id in excluded: + continue + chosen_logprobs[row].append(float(logprobs[row, token_id])) + + out: list[dict[str, Any]] = [] + for row in range(batch_size): + row_logprobs = chosen_logprobs[row] + out.append( + { + "token_entropy": entropies[row] or None, + "avg_logprob": (sum(row_logprobs) / len(row_logprobs)) if row_logprobs else None, + "no_speech_prob": no_speech[row], + } + ) + return out + + +@contextlib.contextmanager +def capture_token_confidence( + pipe: Any, # noqa: ANN401 — a transformers pipeline; typed loosely to avoid a hard import + *, + no_speech_token_id: Optional[int] = None, + special_token_ids: Optional[set[int]] = None, +) -> Iterator[list[dict[str, Any]]]: + """Temporarily wrap ``pipe.model.generate`` to harvest per-token confidence. + + Yields a list that accumulates one confidence dict per generated sequence, in + call order. The wrapper adds ``output_logits`` / ``return_dict_in_generate`` + to every generate call and is removed on exit, leaving the pipeline exactly as + it was found. Backends that ignore those flags (returning a bare tensor) + simply contribute nothing — the caller degrades to ``None`` fields. + + Args: + pipe: A transformers ASR pipeline exposing ``.model.generate``. + no_speech_token_id: Passed through to :func:`token_confidence_from_logits`. + special_token_ids: Passed through to :func:`token_confidence_from_logits`. + + Yields: + The accumulating list of per-sequence confidence dicts. + """ + captured: list[dict[str, Any]] = [] + model = getattr(pipe, "model", None) + if model is None or not callable(getattr(model, "generate", None)): + yield captured + return + + original = model.generate + had_own_attr = "generate" in vars(model) + + def _score(step_logits: Any, sequences: Any) -> list[dict[str, Any]]: # noqa: ANN401 — tensors + return token_confidence_from_logits( + logits=list(step_logits), + sequences=sequences, + no_speech_token_id=no_speech_token_id, + special_token_ids=special_token_ids, + ) + + def _wrapped(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 — passthrough + # Whether the *caller* wanted a dict back. Whisper's word-timestamp path + # does (the pipeline sets return_token_timestamps / return_segments); + # plain greedy decoding does not, and handing those callers a ModelOutput + # where they expect a tensor breaks them downstream + # ("'ModelOutput' object has no attribute 'dtype'"). + caller_wants_dict = bool( + kwargs.get("return_dict_in_generate") + or kwargs.get("return_token_timestamps") + or kwargs.get("return_segments") + ) + try: + result = original(*args, **{**kwargs, "output_logits": True, "return_dict_in_generate": True}) + except TypeError: + # Backend's generate doesn't accept these kwargs — run it untouched. + return original(*args, **kwargs) + try: + step_logits = _get(result, "logits") + sequences = _get(result, "sequences") + if step_logits is not None and sequences is not None: + captured.extend(_score(step_logits, sequences)) + else: + # Whisper long-form (the `return_timestamps="word"` default) reports + # nothing at the top level; each window's generate output lives at + # segments[batch][window]["result"]. Merge a batch item's windows into + # one block so the count still matches one-per-transcript. + segments = _get(result, "segments") + for item in segments or []: + blocks: list[dict[str, Any]] = [] + for window in item or []: + inner = window.get("result") if isinstance(window, dict) else None + if inner is None: + continue + inner_logits = _get(inner, "logits") + inner_sequences = _get(inner, "sequences") + if inner_logits is None or inner_sequences is None: + continue + blocks.extend(_score(inner_logits, inner_sequences)) + if blocks: + captured.append(merge_confidence_blocks(blocks)) + except (RuntimeError, ValueError, IndexError, TypeError, AttributeError, KeyError): + # Confidence is strictly additive — never fail a transcription because + # the logits came back in an unexpected shape. + pass + + # Restore the return type the caller would have seen without our flags, so + # observing the decoder stays invisible to the pipeline. + if not caller_wants_dict: + sequences = _get(result, "sequences") + if isinstance(sequences, torch.Tensor): + return sequences + return result + + model.generate = _wrapped # type: ignore[method-assign] + try: + yield captured + finally: + if had_own_attr: + model.generate = original # type: ignore[method-assign] + else: + with contextlib.suppress(AttributeError): + del model.generate + + +def _get(obj: Any, key: str) -> Any: # noqa: ANN401 — ModelOutput is dict-like *and* attr-like + """Read ``key`` off a transformers ``ModelOutput`` (attribute or mapping).""" + value = getattr(obj, key, None) + if value is None and isinstance(obj, dict): + value = obj.get(key) + return value diff --git a/src/senselab/audio/tasks/speech_to_text_ensemble/__init__.py b/src/senselab/audio/tasks/speech_to_text_ensemble/__init__.py new file mode 100644 index 000000000..a25ef6c7b --- /dev/null +++ b/src/senselab/audio/tasks/speech_to_text_ensemble/__init__.py @@ -0,0 +1,9 @@ +""".. include:: ./doc.md""" # noqa: D415 + +from senselab.audio.tasks.speech_to_text_ensemble.api import ( + fuse_word_streams, + iter_word_leaves, + load_calibrator, +) + +__all__ = ["fuse_word_streams", "iter_word_leaves", "load_calibrator"] diff --git a/src/senselab/audio/tasks/speech_to_text_ensemble/api.py b/src/senselab/audio/tasks/speech_to_text_ensemble/api.py new file mode 100644 index 000000000..eb2d633a7 --- /dev/null +++ b/src/senselab/audio/tasks/speech_to_text_ensemble/api.py @@ -0,0 +1,200 @@ +"""Transcript ensemble fusion: ROVER-style time-slot voting over word streams. + +Model-independent and dependency-free (stdlib only). Callers provide the word +streams (``{system_id: [{"text", "start", "end", "confidence"?}, ...]}``) and an +optional per-system ``weights`` map — e.g. senselab's model-family weights so +that several checkpoints of one architecture don't masquerade as independent +witnesses, or accuracy-derived weights from a validation set. +""" + +from __future__ import annotations + +import math +import re +from typing import Any + +_PUNCTUATION_PATTERN = re.compile(r"[^\w\s']") +_WHITESPACE_PATTERN = re.compile(r"\s+") + + +def _normalize_word(text: str) -> str: + cleaned = _PUNCTUATION_PATTERN.sub(" ", text.lower()) + return _WHITESPACE_PATTERN.sub(" ", cleaned).strip() + + +def iter_word_leaves(node: Any) -> list[dict[str, Any]]: # noqa: ANN401 — recursive JSON walk + """Deepest (text, start, end) leaves of a serialized ``ScriptLine`` tree = words. + + Operates on dicts/lists (JSON artifacts or ``ScriptLine.model_dump()``); for + ``ScriptLine`` *instances* use :meth:`ScriptLine.iter_leaves`. + """ + out: list[dict[str, Any]] = [] + if isinstance(node, list): + for item in node: + out.extend(iter_word_leaves(item)) + return out + if not isinstance(node, dict): + return out + chunks = node.get("chunks") + if isinstance(chunks, list) and chunks: + for c in chunks: + out.extend(iter_word_leaves(c)) + return out + text, start, end = node.get("text"), node.get("start"), node.get("end") + if text and start is not None and end is not None: + try: + word = {"text": str(text).strip(), "start": float(start), "end": float(end)} + except (TypeError, ValueError): + return out + score = node.get("score") + if isinstance(score, (int, float)) and 0.0 <= float(score) <= 1.0: + word["confidence"] = float(score) + if word["text"]: + out.append(word) + return out + + +def load_calibrator(profile: Any) -> Any: # noqa: ANN401 — callable | None + """Build a confidence calibrator from a profile dict. + + Supported shapes: ``{"type": "logistic", "a": float, "b": float}`` applies + ``sigmoid(a · logit(c) + b)``; ``{"type": "piecewise", "x": [...], "y": [...]}`` + linearly interpolates. ``None``/missing → no calibration. + """ + if not profile: + return None + kind = str(profile.get("type") or "") + if kind == "logistic": + a, b = float(profile["a"]), float(profile["b"]) + + def _logistic(c: float) -> float: + c = min(1.0 - 1e-6, max(1e-6, float(c))) + z = a * math.log(c / (1.0 - c)) + b + return round(1.0 / (1.0 + math.exp(-z)), 4) + + return _logistic + if kind == "piecewise": + xs = [float(v) for v in profile["x"]] + ys = [float(v) for v in profile["y"]] + if len(xs) != len(ys) or len(xs) < 2: + raise ValueError("piecewise calibration profile needs matching x/y with >= 2 knots") + + def _piecewise(c: float) -> float: + c = float(c) + if c <= xs[0]: + return round(ys[0], 4) + for i in range(1, len(xs)): + if c <= xs[i]: + t = (c - xs[i - 1]) / max(1e-9, xs[i] - xs[i - 1]) + return round(ys[i - 1] + t * (ys[i] - ys[i - 1]), 4) + return round(ys[-1], 4) + + return _piecewise + raise ValueError(f"unknown calibration profile type: {kind!r}") + + +def fuse_word_streams( + word_streams: dict[str, list[dict[str, Any]]], + *, + weights: dict[str, float] | None = None, + slot_overlap: float = 0.3, + slot_mid_tol_s: float = 0.15, + winner_margin: float = 0.66, + alternate_min_share: float = 0.15, + speaker_at: Any = None, # noqa: ANN401 — callable (t) -> str | None + p_voice_at: Any = None, # noqa: ANN401 — callable (t) -> float | None + calibrator: Any = None, # noqa: ANN401 — callable (c) -> c' | None +) -> list[dict[str, Any]]: + """Fuse per-system word streams into one consensus word list (ROVER-lite). + + Words across systems are grouped into time slots (time-overlap fraction ≥ + ``slot_overlap`` or midpoint distance ≤ ``slot_mid_tol_s``); each slot votes + on the normalized text with ``weights[system] × word confidence``. The + winner's confidence is ``vote share × mean member confidence × coverage``, + where coverage penalizes slots that only a subset of systems witnessed + (abstention is evidence). Alternates are recorded when the winner's share is + below ``winner_margin``. Optional ``speaker_at``/``p_voice_at`` lookups + attribute speakers and flag low-presence words; ``calibrator`` maps raw + confidences through a calibration profile. + + Returns time-ordered ``[{text, start, end, confidence, coverage, sources, + alternates, flags, speaker?}]``. + """ + weights = weights or {} + entries = [] + for system, words in word_streams.items(): + for w in words: + entries.append({**w, "model": system, "mid": (w["start"] + w["end"]) / 2.0}) + entries.sort(key=lambda e: (e["mid"], e["start"], e["model"])) + + slots: list[dict[str, Any]] = [] + for e in entries: + placed = False + for slot in slots: + if e["model"] in slot["models"]: + continue + ov = min(slot["end"], e["end"]) - max(slot["start"], e["start"]) + frac = ov / max(1e-9, min(slot["end"] - slot["start"], e["end"] - e["start"])) + if frac >= slot_overlap or abs(e["mid"] - slot["mid"]) <= slot_mid_tol_s: + slot["members"].append(e) + slot["models"].add(e["model"]) + n = len(slot["members"]) + slot["start"] = sum(m["start"] for m in slot["members"]) / n + slot["end"] = sum(m["end"] for m in slot["members"]) / n + slot["mid"] = (slot["start"] + slot["end"]) / 2.0 + placed = True + break + if not placed: + slots.append( + {"start": e["start"], "end": e["end"], "mid": e["mid"], "members": [e], "models": {e["model"]}} + ) + slots.sort(key=lambda s: (s["start"], s["end"])) + + ensemble_weight = sum(weights.get(m, 1.0) for m in word_streams) or 1.0 + fused: list[dict[str, Any]] = [] + for slot in slots: + tally: dict[str, dict[str, Any]] = {} + total_w = 0.0 + for m in slot["members"]: + key = _normalize_word(m["text"]) or m["text"].lower() + wt = weights.get(m["model"], 1.0) * float(m.get("confidence", 1.0)) + total_w += wt + t = tally.setdefault(key, {"weight": 0.0, "models": [], "display": m["text"], "confs": []}) + t["weight"] += wt + t["models"].append(m["model"]) + if "confidence" in m: + t["confs"].append(m["confidence"]) + if not tally or total_w <= 0: + continue + ranked = sorted(tally.items(), key=lambda kv: (-kv[1]["weight"], kv[0])) + win_key, win = ranked[0] + share = win["weight"] / total_w + member_conf = sum(win["confs"]) / len(win["confs"]) if win["confs"] else 1.0 + coverage = min(1.0, sum(weights.get(m, 1.0) for m in slot["models"]) / ensemble_weight) + raw_conf = share * member_conf * coverage + word = { + "text": win["display"], + "start": round(slot["start"], 4), + "end": round(slot["end"], 4), + "confidence": round(calibrator(raw_conf), 4) if calibrator else round(raw_conf, 4), + "coverage": round(coverage, 4), + "sources": sorted(set(win["models"])), + "alternates": [], + "flags": (["single_source"] if len(slot["models"]) == 1 else []), + } + if share < winner_margin: + for _key, alt in ranked[1:]: + alt_share = alt["weight"] / total_w + if alt_share >= alternate_min_share: + word["alternates"].append( + {"text": alt["display"], "share": round(alt_share, 4), "models": sorted(set(alt["models"]))} + ) + mid = slot["mid"] + if speaker_at is not None: + word["speaker"] = speaker_at(mid) + if p_voice_at is not None: + pv = p_voice_at(mid) + if pv is not None and pv < 0.5: + word["flags"].append("low_presence") + fused.append(word) + return fused diff --git a/src/senselab/audio/tasks/speech_to_text_ensemble/doc.md b/src/senselab/audio/tasks/speech_to_text_ensemble/doc.md new file mode 100644 index 000000000..11cd22f30 --- /dev/null +++ b/src/senselab/audio/tasks/speech_to_text_ensemble/doc.md @@ -0,0 +1,15 @@ +# Speech-to-text ensemble (transcript fusion) + +Combine word-timestamped transcripts from multiple ASR systems into a single +consensus transcript with per-word confidence and alternates (ROVER-style +time-slot voting). Promoted from the adaptive uncertainty workflow +(spec `20260723-225523-dynamic-uncertainty-workflow`, architecture-review T050); +model-independent and dependency-free. + +- `fuse_word_streams(word_streams, weights=...)` — time-slot voting; `weights` + lets callers down-weight correlated systems (e.g. senselab's model-family + weights) or weight by validation accuracy. +- `load_calibrator(profile)` — logistic / piecewise confidence calibration maps. +- `iter_word_leaves(node)` — recursive word-leaf extraction from serialized + `ScriptLine` trees (dicts); for `ScriptLine` *instances* use + `ScriptLine.iter_leaves()`. diff --git a/src/senselab/audio/tasks/speech_to_text_evaluation/utils.py b/src/senselab/audio/tasks/speech_to_text_evaluation/utils.py index caf77f9c9..8dec99fef 100644 --- a/src/senselab/audio/tasks/speech_to_text_evaluation/utils.py +++ b/src/senselab/audio/tasks/speech_to_text_evaluation/utils.py @@ -1,5 +1,7 @@ """This module implements some utilities for evaluating a transcription.""" +import re + try: import jiwer @@ -9,6 +11,29 @@ # TODO: add more metrics which take into account the meaning/intention +_PUNCTUATION_PATTERN = re.compile(r"[^\w\s']") +_WHITESPACE_PATTERN = re.compile(r"\s+") + + +def normalize_transcript_for_wer(text: str) -> str: + """Lowercase, strip non-word punctuation, collapse whitespace. + + The shared surface-normalization applied before WER-style comparisons so + that ``"first."`` vs ``"first!"`` and ``"I"`` vs ``"i"`` don't count as + errors (moved here from the audio-analysis workflow — architecture-review + T049 — so task- and workflow-level WER share one definition). + + Args: + text (str): raw transcript text. + + Returns: + str: normalized text (may be empty). + """ + if not text: + return "" + cleaned = _PUNCTUATION_PATTERN.sub(" ", text.lower()) + return _WHITESPACE_PATTERN.sub(" ", cleaned).strip() + def calculate_wer(reference: str, hypothesis: str) -> float: """Calculate the Word Error Rate (WER) between the reference and hypothesis. diff --git a/src/senselab/audio/tasks/voice_activity_detection/frame_posteriors.py b/src/senselab/audio/tasks/voice_activity_detection/frame_posteriors.py new file mode 100644 index 000000000..3f1c0fb44 --- /dev/null +++ b/src/senselab/audio/tasks/voice_activity_detection/frame_posteriors.py @@ -0,0 +1,317 @@ +"""Continuous per-frame speech posteriors from ``pyannote/segmentation-3.0``. + +Unlike ``detect_human_voice_activity_in_audios`` (which runs the high-level +``Pipeline`` and returns thresholded ``ScriptLine`` segments), this extractor +uses the low-level ``Model`` + ``Inference`` path to obtain the **raw per-frame +speech probability** (~16.9 ms/frame) without any segment thresholding or +hangover smoothing — exactly what the presence axis needs to resolve brief +events and to compute a within-bucket temporal-instability signal. + +``segmentation-3.0`` is a powerset model (up to 3 speakers / chunk); P(speech) +is derived as ``1 − P(no-speaker)``. The model is gated on HuggingFace; loading +reuses ``ensure_hf_model`` + ``get_huggingface_token`` so cached runs skip the +Hub (constitution VI). No new dependency. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional + +import numpy as np + +from senselab.audio.data_structures import Audio +from senselab.utils.data_structures import DeviceType, PyannoteAudioModel, _select_device_and_dtype +from senselab.utils.data_structures.logging import logger +from senselab.utils.data_structures.model import get_huggingface_token +from senselab.utils.dependencies import ensure_hf_model, hf_local_files_only, retry_on_transient_error + +if TYPE_CHECKING: + from pyannote.audio import Inference + +try: + from pyannote.audio import Inference, Model + + PYANNOTEAUDIO_AVAILABLE = True +except (ImportError, RuntimeError): + PYANNOTEAUDIO_AVAILABLE = False + +SEGMENTATION_MODEL_ID = "pyannote/segmentation-3.0" +SEGMENTATION_REVISION = "main" + + +@dataclass +class FramePosterior: + """Continuous per-frame speech probability for one audio. + + Attributes: + probs: ``(num_frames,)`` P(speech) in ``[0, 1]``. + frame_hop_s: seconds between consecutive frame starts. + frame_win_s: analysis window per frame (seconds). + per_class: optional ``(num_frames, num_classes)`` raw class posteriors — + powerset ``[∅, s1, s2, s3, s1s2, s1s3, s2s3]`` for pyannote 3.x + segmentation, or per-speaker multilabel activations for 4.x. Only + populated when ``extract_speech_frame_posteriors(..., + include_per_class=True)`` (spec 20260723-225523 FR-016); ``None`` + otherwise so existing consumers are unaffected. + """ + + probs: np.ndarray + frame_hop_s: float + frame_win_s: float = 0.0 + per_class: Optional[np.ndarray] = None + + def mean_std_in_window(self, start_s: float, end_s: float) -> tuple[float, float]: + """Return ``(mean, std)`` of P(speech) over frames overlapping ``[start, end)``. + + The mean is the bucket's speech-presence contribution; the std captures + within-bucket temporal instability (a bucket straddling an onset has + high frame variance). Returns ``(nan, nan)`` when no frame overlaps. + """ + if self.frame_hop_s <= 0 or self.probs.size == 0: + return (float("nan"), float("nan")) + lo = max(0, int(np.floor(start_s / self.frame_hop_s))) + hi = min(self.probs.size, int(np.ceil(end_s / self.frame_hop_s))) + if hi <= lo: + return (float("nan"), float("nan")) + window = self.probs[lo:hi] + return (float(np.nanmean(window)), float(np.nanstd(window))) + + def overlap_probs(self) -> Optional[np.ndarray]: + """Per-frame P(≥ 2 concurrent speakers) from the per-class posteriors. + + Powerset output (rows summing to ~1, ≥ 5 classes): sum of the + multi-speaker classes (columns 4+). Multilabel per-speaker output + (pyannote 4.x): the second-highest speaker activation. ``None`` when + ``per_class`` was not requested at extraction time. + """ + if self.per_class is None or self.per_class.ndim != 2: + return None + return _overlap_prob_from_output(self.per_class) + + +def _speech_prob_from_output(data: np.ndarray) -> np.ndarray: + """Reduce a segmentation model's per-frame output to P(speech) in ``[0, 1]``. + + ``segmentation-3.0`` emits per-frame powerset class probabilities that sum + to ~1 with class 0 = "no speaker"; P(speech) = ``1 − P(class 0)``. If the + output instead looks multilabel (rows not summing to ~1), fall back to the + max over the class axis. Validated end-to-end in T044. + """ + if data.ndim == 1: + return np.clip(data, 0.0, 1.0) + row_sums = data.sum(axis=1) + if np.nanmean(np.abs(row_sums - 1.0)) < 0.1: # powerset softmax + return np.clip(1.0 - data[:, 0], 0.0, 1.0) + return np.clip(data.max(axis=1), 0.0, 1.0) + + +def _overlap_prob_from_output(data: np.ndarray) -> np.ndarray: + """Reduce per-frame class posteriors to P(overlapping speech) in ``[0, 1]``. + + Mirrors :func:`_speech_prob_from_output`'s powerset-vs-multilabel detection: + powerset softmax (rows ~1) with ≥ 5 classes → sum of multi-speaker classes + (columns 4+); otherwise treat columns as per-speaker activations and take + the second-highest per frame (the probability a second concurrent speaker + is active). + """ + if data.ndim != 2 or data.shape[1] < 2: + return np.zeros(data.shape[0] if data.ndim >= 1 else 0) + row_sums = data.sum(axis=1) + if data.shape[1] >= 5 and float(np.nanmean(np.abs(row_sums - 1.0))) < 0.1: # powerset softmax + return np.clip(data[:, 4:].sum(axis=1), 0.0, 1.0) + sorted_desc = np.sort(data, axis=1)[:, ::-1] + return np.clip(sorted_desc[:, 1], 0.0, 1.0) + + +def _output_to_array(output: Any) -> np.ndarray: # noqa: ANN401 + """Coerce an Inference result to a ``(num_frames, num_classes)`` float array. + + ``window="whole"`` yields a bare numpy array; sliding modes yield a + ``SlidingWindowFeature`` whose frame data lives on ``.data``. + """ + if hasattr(output, "sliding_window") and hasattr(output, "data"): + return np.asarray(output.data, dtype=np.float64) + return np.asarray(output, dtype=np.float64) + + +def _frame_grid(inference: "Inference", num_frames: int, dur_s: float) -> tuple[float, float]: + """Return ``(frame_hop_s, frame_win_s)`` from the model's receptive field. + + Falls back to ``dur_s / num_frames`` (uniform tiling) when the receptive + field isn't introspectable. + """ + try: + rf = inference.model.receptive_field + step, duration = float(rf.step), float(rf.duration) + if step > 0: + return step, duration + except (AttributeError, TypeError, ValueError): + pass + hop = dur_s / max(1, num_frames) + return hop, hop + + +# segmentation-3.0 (and Brouhaha) train on 10 s chunks. For recordings longer +# than one chunk we slide a bounded window and stitch, so memory stays flat and +# we avoid pyannote's "whole-file on a frame-based model" degradation, while +# keeping native ~17 ms frame resolution. +_CHUNK_S = 10.0 +_CHUNK_STEP_S = 8.0 # 2 s overlap → smooth stitching across chunk seams + + +def stitch_frames( + chunk_arrays: list[np.ndarray], + chunk_starts_s: list[float], + hop_s: float, +) -> np.ndarray: + """Overlap-average per-chunk ``(frames, C)`` arrays into one continuous timeline. + + Each chunk's frame ``i`` maps to absolute frame index ``round(start/hop) + i``; + indices covered by multiple (overlapping) chunks are averaged. Trailing + uncovered frames are trimmed. Returns a ``(num_frames, C)`` array. Pure / + model-free so it is shared by the in-process extractor and the Brouhaha + subprocess path. + """ + if not chunk_arrays or hop_s <= 0: + return np.zeros((0, 1)) + norm = [a[:, None] if a.ndim == 1 else a for a in chunk_arrays] + n_classes = norm[0].shape[1] + n_global = 0 + for arr, t0 in zip(norm, chunk_starts_s): + n_global = max(n_global, int(round(t0 / hop_s)) + arr.shape[0]) + accum = np.zeros((n_global, n_classes), dtype=np.float64) + count = np.zeros(n_global, dtype=np.float64) + for arr, t0 in zip(norm, chunk_starts_s): + base = int(round(t0 / hop_s)) + for i in range(arr.shape[0]): + g = base + i + if 0 <= g < n_global: + accum[g] += arr[i] + count[g] += 1 + covered = np.nonzero(count > 0)[0] + last = int(covered[-1]) + 1 if covered.size else 0 + return accum[:last] / np.maximum(count[:last, None], 1.0) + + +def chunked_frame_inference( + inference: "Inference", + audio: Audio, + chunk_s: float = _CHUNK_S, + step_s: float = _CHUNK_STEP_S, +) -> tuple[np.ndarray, float, float]: + """Run per-frame inference over the whole recording, chunking long ones. + + For clips at or under ``chunk_s`` this is a single ``window="whole"`` pass. + For longer recordings it slides overlapping ``chunk_s`` windows (each a + bounded whole-window pass), maps every chunk-local frame to an absolute + frame index, and averages the overlaps — yielding one continuous + ``(num_frames, num_classes)`` array at native frame resolution with flat + memory. Returns ``(data, frame_hop_s, frame_win_s)``. + """ + sr = int(audio.sampling_rate) + total = int(audio.waveform.shape[-1]) + dur = total / sr if sr else 0.0 + hop, win = _frame_grid(inference, 1, max(dur, 1e-9)) + + if dur <= chunk_s or hop <= 0: + arr = _output_to_array(inference({"waveform": audio.waveform, "sample_rate": sr})) + if arr.ndim == 1: + arr = arr[:, None] + h, w = _frame_grid(inference, arr.shape[0], max(dur, 1e-9)) + return arr, h, w + + chunk_samples = int(chunk_s * sr) + step_samples = max(1, int(step_s * sr)) + chunk_arrays: list[np.ndarray] = [] + chunk_starts_s: list[float] = [] + start = 0 + while start < total: + end = min(total, start + chunk_samples) + if end - start < int(0.1 * sr): # skip a sub-100ms tail sliver + break + arr = _output_to_array(inference({"waveform": audio.waveform[:, start:end], "sample_rate": sr})) + chunk_arrays.append(arr) + chunk_starts_s.append(start / sr) + if end >= total: + break + start += step_samples + + data = stitch_frames(chunk_arrays, chunk_starts_s, hop) + return data, hop, win + + +_inference_cache: dict[str, "Inference"] = {} + + +def _get_inference(model: PyannoteAudioModel, device: Optional[DeviceType]) -> "Inference": + """Load (and cache) a segmentation ``Inference`` for the requested model/device.""" + import torch + + device, _ = _select_device_and_dtype(user_preference=device, compatible_devices=[DeviceType.CUDA, DeviceType.CPU]) + key = f"{model.path_or_uri}-{model.revision}-{device}" + if key not in _inference_cache: + ensure_hf_model(str(model.path_or_uri), revision=model.revision, token=get_huggingface_token()) + loaded = retry_on_transient_error( + Model.from_pretrained, + model.path_or_uri, + revision=model.revision, + token=get_huggingface_token(), + ) + if loaded is None: + raise ValueError(f"segmentation model {model.path_or_uri} could not be loaded.") + # window="whole" returns the model's NATIVE per-frame posterior (~17 ms/frame) + # for the entire signal. The default (sliding) mode returns a coarse + # per-chunk aggregate (~1 s step) that would defeat the fine presence grid. + # Trade-off: "whole" holds the whole signal in one forward pass — fine for + # the short clinical clips this workflow targets; very long recordings would + # need chunked stitching (future work). + _inference_cache[key] = Inference(loaded, window="whole", device=torch.device(device.value)) + return _inference_cache[key] + + +def extract_speech_frame_posteriors( + audios: list[Audio], + model: Optional[PyannoteAudioModel] = None, + device: Optional[DeviceType] = None, + include_per_class: bool = False, +) -> list[Optional[FramePosterior]]: + """Return continuous per-frame P(speech) for each audio (``None`` on failure). + + If the model cannot be loaded (not installed, gated without a token, + native-lib failure), every entry is ``None`` and the presence axis simply + omits the frame-posterior voter (FR-023) — the workflow does not abort. + + With ``include_per_class=True`` the raw ``(frames, classes)`` posteriors are + retained on ``FramePosterior.per_class`` (enabling + :meth:`FramePosterior.overlap_probs` — FR-016); default ``False`` keeps the + historical memory profile and output shape. + """ + if not PYANNOTEAUDIO_AVAILABLE: + logger.warning("pyannote-audio unavailable; segmentation frame posteriors will be null.") + return [None] * len(audios) + try: + # PyannoteAudioModel validates id/revision against HF at construction + # (ValidationError for a gated repo the token can't access), so guard it too. + if model is None: + model = PyannoteAudioModel(path_or_uri=SEGMENTATION_MODEL_ID, revision=SEGMENTATION_REVISION) + hf_local_files_only(str(model.path_or_uri), revision=model.revision) + inference = _get_inference(model=model, device=device) + except Exception as exc: # noqa: BLE001 — any load/access failure degrades to null (FR-023) + logger.warning(f"Failed to load {SEGMENTATION_MODEL_ID}: {exc}. Frame posteriors will be null.") + return [None] * len(audios) + + results: list[Optional[FramePosterior]] = [] + for audio in audios: + try: + t0 = time.time() + data, hop_s, win_s = chunked_frame_inference(inference, audio) + probs = _speech_prob_from_output(data) + per_class = np.asarray(data, dtype=np.float64) if include_per_class and data.ndim == 2 else None + results.append(FramePosterior(probs=probs, frame_hop_s=hop_s, frame_win_s=win_s, per_class=per_class)) + logger.info(f"segmentation inference took {time.time() - t0:.2f}s ({probs.size} frames @ {hop_s:.4f}s)") + except (RuntimeError, ValueError, OSError) as exc: + logger.warning(f"segmentation inference failed for one audio: {exc}") + results.append(None) + return results diff --git a/src/senselab/audio/workflows/__init__.py b/src/senselab/audio/workflows/__init__.py index 843f38314..43cdac95f 100644 --- a/src/senselab/audio/workflows/__init__.py +++ b/src/senselab/audio/workflows/__init__.py @@ -1,5 +1,25 @@ -"""Workflows and pipelines for audio processing and analysis.""" +"""Workflows and pipelines for audio processing and analysis. -from .explore_conversation import explore_conversation # noqa: F401 +``explore_conversation`` is resolved lazily (PEP 562): importing this package — +or any pure submodule under ``audio_analysis`` — must not pull the four model +task stacks that ``explore_conversation`` depends on (architecture-review.md +F1 / T046). +""" + +from typing import Any __all__ = ["explore_conversation"] + + +def __getattr__(name: str) -> Any: # noqa: ANN401 — lazy re-export + """Resolve ``explore_conversation`` on first access.""" + if name == "explore_conversation": + from senselab.audio.workflows.explore_conversation import explore_conversation + + return explore_conversation + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + """Expose lazy exports to introspection / pdoc.""" + return __all__ diff --git a/src/senselab/audio/workflows/audio_analysis/__init__.py b/src/senselab/audio/workflows/audio_analysis/__init__.py index 03eb1d372..bf5c5e210 100644 --- a/src/senselab/audio/workflows/audio_analysis/__init__.py +++ b/src/senselab/audio/workflows/audio_analysis/__init__.py @@ -10,44 +10,52 @@ wrapper, but it is also importable standalone: from senselab.audio.workflows.audio_analysis import compute_uncertainty_axes + +Public symbols are resolved lazily (PEP 562, same pattern as ``adaptive/``) so +that importing the *pure* submodules (``aggregate``, ``aggregators``, ``grid``, +``types``, ``votes``, ``harvesters``, ``adaptive.*``) never pulls +torch / speechbrain / transformers — those load only when a model-touching +symbol (``compute_uncertainty_axes``, ``extract_per_window_embeddings``, the +plot) is actually requested (architecture-review.md F1 / T046). """ -from senselab.audio.workflows.audio_analysis.aggregators import ( - AGGREGATORS, - apply_aggregator, -) -from senselab.audio.workflows.audio_analysis.compute import compute_uncertainty_axes -from senselab.audio.workflows.audio_analysis.disagreements import build_disagreements_index -from senselab.audio.workflows.audio_analysis.embeddings import ( - WindowEmbedding, - extract_per_window_embeddings, -) -from senselab.audio.workflows.audio_analysis.grid import BucketGrid -from senselab.audio.workflows.audio_analysis.io import write_axis_parquet -from senselab.audio.workflows.audio_analysis.labelstudio import ( - attach_uncertainty_tracks_to_ls, - uncertainty_to_label_bin, -) -from senselab.audio.workflows.audio_analysis.plot import build_aligned_timeline_plot -from senselab.audio.workflows.audio_analysis.types import ( - AxisResult, - UncertaintyAxis, - UncertaintyRow, -) - -__all__ = [ - "AGGREGATORS", - "AxisResult", - "BucketGrid", - "UncertaintyAxis", - "UncertaintyRow", - "WindowEmbedding", - "apply_aggregator", - "attach_uncertainty_tracks_to_ls", - "build_aligned_timeline_plot", - "build_disagreements_index", - "compute_uncertainty_axes", - "extract_per_window_embeddings", - "uncertainty_to_label_bin", - "write_axis_parquet", -] +from typing import Any + +# symbol name → defining submodule (all imports deferred to first attribute access). +_LAZY_EXPORTS = { + "AGGREGATORS": "aggregators", + "apply_aggregator": "aggregators", + "compute_uncertainty_axes": "compute", + "harvest_pass": "compute", + "build_disagreements_index": "disagreements", + "WindowEmbedding": "embeddings", + "extract_per_window_embeddings": "embeddings", + "BucketGrid": "grid", + "write_axis_parquet": "io", + "attach_uncertainty_tracks_to_ls": "labelstudio", + "uncertainty_to_label_bin": "labelstudio", + "build_aligned_timeline_plot": "plot", + "AxisResult": "types", + "UncertaintyAxis": "types", + "UncertaintyRow": "types", + "PassHarvest": "votes", + "aggregate_pass": "votes", + "compute_pass_deltas": "votes", +} + +__all__ = sorted(_LAZY_EXPORTS) + + +def __getattr__(name: str) -> Any: # noqa: ANN401 — lazy re-export + """Resolve public symbols on first access without importing heavy submodules.""" + submodule = _LAZY_EXPORTS.get(name) + if submodule is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + return getattr(import_module(f"{__name__}.{submodule}"), name) + + +def __dir__() -> list[str]: + """Expose lazy exports to introspection / pdoc.""" + return __all__ diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/__init__.py b/src/senselab/audio/workflows/audio_analysis/adaptive/__init__.py new file mode 100644 index 000000000..c1ffb69f5 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/__init__.py @@ -0,0 +1,27 @@ +"""Uncertainty-driven adaptive analysis loop (prototype). + +Spec: ``specs/20260723-225523-dynamic-uncertainty-workflow/``. + +This subpackage keeps imports light on purpose: no torch / model backends are +imported at module level, so the loop's pure core (belief store, region +proposal, policy engine, fusion, evaluation) runs in minimal environments. +Interventions that need live model backends import them lazily inside their +``execute`` functions and degrade to ``blocked_guard`` when unavailable. + +Public API:: + + from senselab.audio.workflows.audio_analysis.adaptive import run_adaptive_loop +""" + +from typing import Any + +__all__ = ["run_adaptive_loop"] + + +def __getattr__(name: str) -> Any: # noqa: ANN401 — lazy re-export + """Lazily resolve public symbols so importing the package stays light.""" + if name == "run_adaptive_loop": + from senselab.audio.workflows.audio_analysis.adaptive.loop import run_adaptive_loop + + return run_adaptive_loop + raise AttributeError(name) diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/audio_io.py b/src/senselab/audio/workflows/audio_analysis/adaptive/audio_io.py new file mode 100644 index 000000000..ca6dd3968 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/audio_io.py @@ -0,0 +1,176 @@ +"""Minimal audio access for live interventions (no senselab data-structure deps). + +Loads the run's input audio as 16 kHz mono float32, crops regions, and +regenerates the enhanced stream on demand (SepFormer) when a live backend is +available. Kept dependency-light so the loop degrades gracefully: every entry +point returns ``(payload, None)`` or ``(None, reason)``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +TARGET_SR = 16000 + + +def load_wav_16k_mono( + path: Path, *, backend: str = "auto", backend_out: dict[str, Any] | None = None +) -> tuple[Any | None, str | None]: # noqa: ANN401 — np.ndarray + """Read ``path`` → (float32 mono 16 kHz ndarray, None) or (None, reason). + + ``backend`` (policy ``audio_io_backend``): + + - ``"auto"`` (default): the senselab preprocessing path first — replicating + analyze_audio's ``prepare_audio`` exactly (``read_audios → + downmix_audios_to_mono → resample_audios``) so crops share + ``audio_signature``/cache entries with pipeline-produced crops + (architecture-review.md F3/T048) — then the soundfile+polyphase DSP + fallback for artifact-driven runs outside a senselab install (different + resampler ⇒ different signatures; results valid, not cache-shareable). + - ``"senselab"``: strict — no fallback; fail loudly when the preprocessing + stack is unavailable (recommended in production). + - ``"dsp"``: pin the fallback loader (skips the senselab attempt entirely — + useful where that stack is known-broken or too slow to probe). + + Never silent: the loader used lands in ``backend_out["loader"]`` and is + surfaced in run provenance (``convergence.json → audio_backend``). + """ + reason: str | None = None + if backend in ("auto", "senselab"): + wav, reason = _load_senselab(path) + if wav is not None: + if backend_out is not None: + backend_out["loader"] = "senselab.preprocessing" + return wav, None + if backend == "senselab": + return None, f"audio_fallback_forbidden ({reason})" + wav, fb_reason = _load_fallback(path, senselab_reason=reason) + if wav is not None and backend_out is not None: + backend_out["loader"] = "dsp_fallback" + if reason is not None: + backend_out["senselab_path_reason"] = reason + return wav, fb_reason + + +def _load_senselab(path: Path) -> tuple[Any | None, str | None]: # noqa: ANN401 + try: + from senselab.audio.tasks.input_output import read_audios # noqa: PLC0415 + from senselab.audio.tasks.preprocessing import ( # noqa: PLC0415 + downmix_audios_to_mono, + resample_audios, + ) + except ImportError as exc: + return None, f"senselab_audio_unavailable ({getattr(exc, 'name', exc)})" + try: + import numpy as np # noqa: PLC0415 + + audio = read_audios([str(path)])[0] + audio = downmix_audios_to_mono([audio])[0] + if audio.sampling_rate != TARGET_SR: + audio = resample_audios([audio], resample_rate=TARGET_SR)[0] + wav = audio.waveform.detach().cpu().numpy().squeeze() + return np.ascontiguousarray(wav, dtype="float32"), None + except Exception as exc: # noqa: BLE001 + return None, f"senselab_audio_failed ({exc!r})" + + +def _load_fallback(path: Path, *, senselab_reason: str | None) -> tuple[Any | None, str | None]: # noqa: ANN401 + try: + import numpy as np + import soundfile as sf + except ImportError as exc: + return None, f"audio_io_unavailable ({exc.name}; senselab path: {senselab_reason})" + try: + data, sr = sf.read(str(path), dtype="float32", always_2d=True) + except (OSError, RuntimeError) as exc: + return None, f"audio_read_failed ({exc!r})" + mono = data.mean(axis=1) + if sr != TARGET_SR: + try: + from scipy.signal import resample_poly + except ImportError: + return None, f"audio_io_unavailable (scipy; senselab path: {senselab_reason})" + from math import gcd + + g = gcd(int(sr), TARGET_SR) + mono = resample_poly(mono, TARGET_SR // g, int(sr) // g).astype("float32") + return np.ascontiguousarray(mono, dtype="float32"), None + + +def crop(wav: Any, start_s: float, end_s: float) -> Any: # noqa: ANN401 + """Slice ``[start_s, end_s)`` (clipped); timestamps of outputs must add ``start_s`` back.""" + lo = max(0, int(round(start_s * TARGET_SR))) + hi = min(len(wav), int(round(end_s * TARGET_SR))) + return wav[lo:hi] + + +def get_stream_wav(ctx: dict[str, Any], stream: str) -> tuple[Any | None, str | None]: # noqa: ANN401 + """Waveform for a pass: raw loads the input file; enhanced is regenerated on demand. + + Results are cached in ``ctx["_wav_cache"]``. The enhanced stream mirrors + analyze_audio's whole-file SepFormer pass (research.md D4); when the + backend is unavailable the caller decides whether raw is an acceptable + fallback (recorded as ``stream_fallback`` in the intervention log). + """ + cache = ctx.setdefault("_wav_cache", {}) + if stream in cache: + return cache[stream] + input_audio = ctx.get("input_audio") + io_backend = str((ctx.get("policy") or {}).get("audio_io_backend", "auto")) + backend_out: dict[str, Any] = {} + result: tuple[Any | None, str | None] + if not input_audio or not Path(input_audio).exists(): + result = (None, "input_audio_missing") + elif stream == "raw_16k": + result = load_wav_16k_mono(Path(input_audio), backend=io_backend, backend_out=backend_out) + if backend_out: + ctx.setdefault("audio_backend", {})[stream] = backend_out + elif stream == "enhanced_16k": + raw, reason = get_stream_wav(ctx, "raw_16k") + result = (None, reason) if raw is None else _enhance(raw, ctx) + else: + result = (None, f"unknown_stream ({stream})") + cache[stream] = result + return result + + +def _enhance(wav: Any, ctx: dict[str, Any]) -> tuple[Any | None, str | None]: # noqa: ANN401 + """Whole-file enhancement, routed through ``tasks/speech_enhancement`` (T048). + + Mirrors analyze_audio's enhanced pass (same default model, same task API) so + the regenerated stream matches pipeline semantics; the direct speechbrain + call remains only as the degraded-environment fallback. + """ + model_id = ctx["policy"].get("enhancement_model", "speechbrain/sepformer-wham16k-enhancement") + try: + import numpy as np # noqa: PLC0415 + import torch # noqa: PLC0415 + + from senselab.audio.data_structures import Audio # noqa: PLC0415 + from senselab.audio.tasks.speech_enhancement import enhance_audios # noqa: PLC0415 + from senselab.utils.data_structures import SpeechBrainModel # noqa: PLC0415 + + audio = Audio(waveform=torch.from_numpy(wav).unsqueeze(0), sampling_rate=TARGET_SR) + enhanced = enhance_audios([audio], model=SpeechBrainModel(path_or_uri=model_id))[0] + return np.ascontiguousarray(enhanced.waveform.detach().cpu().numpy().squeeze(), dtype="float32"), None + except ImportError as exc: + senselab_reason = f"senselab_enhancement_unavailable ({getattr(exc, 'name', exc)})" + except Exception as exc: # noqa: BLE001 + senselab_reason = f"senselab_enhancement_failed ({exc!r})" + try: + import torch + from speechbrain.inference.separation import SepformerSeparation + except ImportError as exc: + return None, f"enhancement_backend_unavailable ({getattr(exc, 'name', exc)}; {senselab_reason})" + try: + model = SepformerSeparation.from_hparams(source=model_id, run_opts={"device": "cpu"}) + with torch.no_grad(): + est = model.separate_batch(torch.from_numpy(wav).unsqueeze(0)) + out = est[0, :, 0].cpu().numpy().astype("float32") + peak = max(1e-9, float(abs(out).max())) + if peak > 1.0: + out = out / peak + return out, None + except Exception as exc: # noqa: BLE001 — model/download failures degrade to a reason + return None, f"enhancement_failed ({exc!r}; {senselab_reason})" diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/backends.py b/src/senselab/audio/workflows/audio_analysis/adaptive/backends.py new file mode 100644 index 000000000..7dd970372 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/backends.py @@ -0,0 +1,249 @@ +"""Guarded gateways from the adaptive loop to senselab's task APIs. + +Post-T047 (architecture-review.md F2) this module no longer *implements* model +capabilities — every function delegates to the owning task/workflow API and +exists only to provide the loop's failure envelope: lazy imports, a +``(result, reason)`` return instead of exceptions, crop/offset bookkeeping, and +an explicit degraded-environment fallback where one is justified. + +- U1 re-ASR → ``senselab.audio.tasks.speech_to_text.transcribe_audios`` + (word-level timestamps via ``return_timestamps="word"``); a bare HF-pipeline + fallback remains for environments where the senselab task stack cannot import. +- U3 consensus alignment → ``senselab.audio.tasks.forced_alignment.mms_fa``. +- I1/I2 fine-hop embeddings → the workflow's own + ``audio_analysis.embeddings.extract_per_window_embeddings``. +- I4 overlap posteriors → ``voice_activity_detection.frame_posteriors`` + (``include_per_class=True`` + ``FramePosterior.overlap_probs`` — FR-016). + +Nothing here is file- or model-id-specific: models, windows, and hops come from +the policy. +""" + +from __future__ import annotations + +from typing import Any + +from senselab.audio.workflows.audio_analysis.adaptive.audio_io import TARGET_SR + +_ASR_PIPELINE_CACHE: dict[str, Any] = {} + + +def _to_audio(wav_crop: Any) -> Any: # noqa: ANN401 — returns senselab Audio + """Wrap a 1-D float32 numpy crop as a 16 kHz mono senselab ``Audio``.""" + import torch # noqa: PLC0415 + + from senselab.audio.data_structures import Audio # noqa: PLC0415 + + return Audio(waveform=torch.from_numpy(wav_crop).unsqueeze(0), sampling_rate=TARGET_SR) + + +# ── U1: region re-ASR ──────────────────────────────────────────────────── + + +def transcribe_crop( + wav_crop: Any, # noqa: ANN401 — np.ndarray + *, + model_id: str, + offset_s: float, + language: str | None = None, + meta: dict[str, Any] | None = None, + backend: str = "auto", +) -> tuple[list[dict[str, Any]] | None, str | None]: + """Word-timestamped transcription of one crop, in FILE time (offset applied). + + ``backend`` (policy ``u1_backend``): ``"auto"`` (default) tries the senselab + speech-to-text task API first (arbitrary ``HFModel`` ids, + ``return_timestamps="word"``) and falls back to a bare HF pipeline; + ``"senselab"`` / ``"pipeline"`` pin one path. The backend actually used is + recorded in ``meta["backend"]`` (never silent). + """ + reason: str | None = None + if backend in ("auto", "senselab"): + words, reason = _transcribe_crop_senselab(wav_crop, model_id=model_id, offset_s=offset_s) + if words is not None: + if meta is not None: + meta["backend"] = "senselab.speech_to_text" + return words, None + if backend == "senselab": + return None, reason + fallback_words, fb_reason = _transcribe_crop_pipeline( + wav_crop, model_id=model_id, offset_s=offset_s, language=language + ) + if fallback_words is not None: + if meta is not None: + meta["backend"] = "hf_pipeline" if backend == "pipeline" else "hf_pipeline_fallback" + if reason is not None: + meta["senselab_path_reason"] = reason + return fallback_words, None + return None, f"{reason}; fallback: {fb_reason}" if reason else fb_reason + + +def _transcribe_crop_senselab( + wav_crop: Any, # noqa: ANN401 + *, + model_id: str, + offset_s: float, +) -> tuple[list[dict[str, Any]] | None, str | None]: + try: + from senselab.audio.tasks.speech_to_text import transcribe_audios # noqa: PLC0415 + from senselab.utils.data_structures import HFModel # noqa: PLC0415 + except ImportError as exc: + return None, f"senselab_asr_unavailable ({getattr(exc, 'name', exc)})" + try: + lines = transcribe_audios([_to_audio(wav_crop)], model=HFModel(path_or_uri=model_id), return_timestamps="word") + except Exception as exc: # noqa: BLE001 + return None, f"senselab_asr_failed ({exc!r})" + from senselab.audio.workflows.audio_analysis.adaptive.fusion import iter_word_leaves # noqa: PLC0415 + + serialized = [line.model_dump() if hasattr(line, "model_dump") else line for line in lines or []] + words = [ + {"text": w["text"], "start": round(w["start"] + offset_s, 4), "end": round(w["end"] + offset_s, 4)} + for w in iter_word_leaves(serialized) + ] + return words, None + + +def _transcribe_crop_pipeline( + wav_crop: Any, # noqa: ANN401 + *, + model_id: str, + offset_s: float, + language: str | None = None, +) -> tuple[list[dict[str, Any]] | None, str | None]: + try: + from transformers import pipeline # noqa: PLC0415 + except ImportError as exc: + return None, f"asr_backend_unavailable ({getattr(exc, 'name', exc)})" + try: + if model_id not in _ASR_PIPELINE_CACHE: + _ASR_PIPELINE_CACHE[model_id] = pipeline("automatic-speech-recognition", model=model_id, device="cpu") + asr = _ASR_PIPELINE_CACHE[model_id] + kwargs: dict[str, Any] = {"return_timestamps": "word"} + if language: + kwargs["generate_kwargs"] = {"language": language} + out = asr({"raw": wav_crop, "sampling_rate": TARGET_SR}, **kwargs) + except Exception as exc: # noqa: BLE001 + return None, f"asr_failed ({exc!r})" + words: list[dict[str, Any]] = [] + for ch in out.get("chunks") or []: + ts = ch.get("timestamp") or (None, None) + text = (ch.get("text") or "").strip() + if not text or ts[0] is None: + continue + end = ts[1] if ts[1] is not None else ts[0] + words.append({"text": text, "start": round(float(ts[0]) + offset_s, 4), "end": round(float(end) + offset_s, 4)}) + return words, None + + +# ── I1/I2: fine-hop speaker embeddings ─────────────────────────────────── + + +def embed_windows( + wav: Any, # noqa: ANN401 + *, + model_id: str, + span: tuple[float, float], + win_s: float, + hop_s: float, +) -> tuple[list[dict[str, Any]] | None, str | None]: + """Fine-hop speaker embeddings over ``span`` → ``[{start_s, end_s, vector}]`` (file time). + + Delegates to the workflow's uniform-grid extractor (which itself routes + through ``tasks/speaker_embeddings``) on the cropped waveform. + """ + try: + from senselab.audio.workflows.audio_analysis.embeddings import ( # noqa: PLC0415 + extract_per_window_embeddings, + ) + except ImportError as exc: + return None, f"embedding_backend_unavailable ({getattr(exc, 'name', exc)})" + try: + lo, hi = int(round(span[0] * TARGET_SR)), int(round(span[1] * TARGET_SR)) + failures: dict[str, str] = {} + per_model = extract_per_window_embeddings( + audio=_to_audio(wav[lo:hi]), models=[model_id], window_s=win_s, hop_s=hop_s, failures=failures + ) + windows = per_model.get(model_id) or [] + if not windows: + return None, f"embedding_failed ({failures.get(model_id, 'no windows produced')})" + return [ + { + "start_s": round(float(w.start_s) + span[0], 4), + "end_s": round(float(w.end_s) + span[0], 4), + "vector": [float(x) for x in w.vector.tolist()], + } + for w in windows + ], None + except Exception as exc: # noqa: BLE001 + return None, f"embedding_failed ({exc!r})" + + +# ── I4: overlap posteriors ─────────────────────────────────────────────── + + +def overlap_posteriors( + wav: Any, # noqa: ANN401 + *, + span: tuple[float, float], +) -> tuple[dict[str, Any] | None, str | None]: + """Per-class segmentation posteriors over ``span`` → speech + overlap tracks (FR-016). + + Delegates to ``extract_speech_frame_posteriors(include_per_class=True)`` and + ``FramePosterior.overlap_probs()``; frames are span-local. + """ + import os # noqa: PLC0415 + + if not (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")): + return None, "posteriors_unavailable (HF token required for pyannote/segmentation-3.0)" + try: + from senselab.audio.tasks.voice_activity_detection.frame_posteriors import ( # noqa: PLC0415 + extract_speech_frame_posteriors, + ) + except ImportError as exc: + return None, f"posteriors_unavailable ({getattr(exc, 'name', exc)})" + try: + lo, hi = int(round(span[0] * TARGET_SR)), int(round(span[1] * TARGET_SR)) + fp = extract_speech_frame_posteriors([_to_audio(wav[lo:hi])], include_per_class=True)[0] + except Exception as exc: # noqa: BLE001 + return None, f"posteriors_failed ({exc!r})" + if fp is None: + return None, "posteriors_unavailable (model load/access failed — see logs)" + overlap = fp.overlap_probs() + if overlap is None: + return None, "posteriors_unexpected_shape (per-class posteriors missing)" + return { + "frame_hop": float(fp.frame_hop_s), + "overlap": [float(x) for x in overlap], + "speech": [float(x) for x in fp.probs], + "n_classes": int(fp.per_class.shape[1]) if fp.per_class is not None else None, + }, None + + +# ── U3: consensus word re-alignment ────────────────────────────────────── + + +def consensus_align( + wav: Any, # noqa: ANN401 + words: list[dict[str, Any]], + *, + timeout_s: float = 600.0, +) -> tuple[list[dict[str, Any]] | None, str | None]: + """U3 (C8): align the consensus word sequence via the forced-alignment task's MMS_FA backend.""" + try: + from senselab.audio.tasks.forced_alignment.mms_fa import align_words_mms_fa # noqa: PLC0415 + except ImportError as exc: + return None, f"aligner_backend_unavailable ({getattr(exc, 'name', exc)})" + return align_words_mms_fa(wav, [w["text"] for w in words], timeout_s=timeout_s) + + +def senselab_transcribe_available() -> bool: + """True when the full senselab ASR stack is importable (U1 primary path).""" + try: + import importlib.util + + return ( + importlib.util.find_spec("torch") is not None + and importlib.util.find_spec("senselab.audio.tasks.speech_to_text") is not None + ) + except (ImportError, ValueError, ModuleNotFoundError): + return False diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/belief.py b/src/senselab/audio/workflows/audio_analysis/adaptive/belief.py new file mode 100644 index 000000000..ea432b04b --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/belief.py @@ -0,0 +1,436 @@ +"""Belief store: provenance-tagged votes + re-aggregation (prototype). + +Implements the VoteStore / BeliefRow semantics of +``specs/20260723-225523-dynamic-uncertainty-workflow/contracts/belief-store.md``: + +- one *vote* per (axis, bucket, source, stream, scope) with status + ``active | shadowed | purged_hallucination``; +- region-scoped votes shadow file-scoped votes of the same (source, stream); +- aggregation is a pure function of the active votes, delegated to the + existing per-axis aggregators (``aggregate.py``) — the harvest/aggregate + split (research.md D8) demonstrated on real artifacts. + +The prototype ingests a completed ``analyze_audio`` run directory: the six +per-pass uncertainty parquets are the round-1 vote population, and the stored +``aggregated_uncertainty`` doubles as a parity oracle for the re-aggregation +path (tasks.md T007). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from senselab.audio.workflows.audio_analysis.aggregate import ( + aggregate_identity, + aggregate_presence, + aggregate_utterance, + presence_p_voice, +) + +AXES = ("presence", "identity", "utterance") + +_META_COLUMNS = ( + "presence_confidence", + "presence_uncertainty", + "quality_snr", + "quality_clip", + "quality_reverb", + "quality_bandwidth", + "quality_uncertainty", + "src_speech", + "src_people", + "src_machine", + "src_environment", + "src_dominant", + "token_entropy", + "scene_quality_coupling", + "intensity_weight", + "raw_aggregated_uncertainty", + "comparison_status", +) + + +def bucket_key(start: float, end: float) -> tuple[float, float]: + """Canonical (start, end) bucket key, rounded for float-stable dict use.""" + return (round(float(start), 6), round(float(end), 6)) + + +@dataclass +class Vote: + """One source's statement about one bucket on one axis (data-model.md).""" + + axis: str + bucket: tuple[float, float] + source: str + stream: str + scope: str # "file" | "region:" + round: int + payload: dict[str, Any] + status: str = "active" # active | shadowed | purged_hallucination + shadowed_by: str | None = None + provenance: dict[str, Any] = field(default_factory=dict) + + @property + def vote_id(self) -> str: + """Deterministic id — same (axis, bucket, source, stream, scope) overwrites itself.""" + raw = f"{self.axis}|{self.bucket[0]}|{self.bucket[1]}|{self.source}|{self.stream}|{self.scope}" + return hashlib.sha1(raw.encode()).hexdigest()[:16] + + def to_record(self) -> dict[str, Any]: + """Flat dict for parquet/JSON persistence.""" + return { + "vote_id": self.vote_id, + "axis": self.axis, + "bucket_start": self.bucket[0], + "bucket_end": self.bucket[1], + "source": self.source, + "stream": self.stream, + "scope": self.scope, + "round": self.round, + "status": self.status, + "shadowed_by": self.shadowed_by, + "payload": json.dumps(self.payload, default=str), + "provenance": json.dumps(self.provenance, default=str), + } + + +class VoteStore: + """All evidence for one run, indexed by (stream, axis, bucket).""" + + def __init__(self) -> None: + """Create an empty store.""" + self._votes: dict[str, Vote] = {} + self._index: dict[tuple[str, str, tuple[float, float]], list[str]] = {} + # Per-(stream, axis, bucket) row metadata from the ingested parquets + # (quality / source-mass / presence columns + the stored aggregate used + # as the round-1 parity oracle). + self.row_meta: dict[tuple[str, str, tuple[float, float]], dict[str, Any]] = {} + self._round_added: dict[int, list[str]] = {} + + # ── ingest ───────────────────────────────────────────────────────── + + @classmethod + def from_run_dir(cls, run_dir: Path, passes: list[str]) -> "VoteStore": + """Populate round-1 votes from ``//uncertainty/.parquet``.""" + import pandas as pd + + store = cls() + for stream in passes: + for axis in AXES: + pq = run_dir / stream / "uncertainty" / f"{axis}.parquet" + if not pq.exists(): + continue + df = pd.read_parquet(pq) + for _, row in df.iterrows(): + bk = bucket_key(row["start"], row["end"]) + votes_raw = row.get("model_votes") + if isinstance(votes_raw, str): + try: + votes = json.loads(votes_raw) + except json.JSONDecodeError: + votes = {} + elif isinstance(votes_raw, dict): + votes = votes_raw + else: + votes = {} + for source, payload in (votes or {}).items(): + if not isinstance(payload, dict): + continue + store.add_vote( + Vote( + axis=axis, + bucket=bk, + source=str(source), + stream=stream, + scope="file", + round=1, + payload=payload, + ) + ) + meta: dict[str, Any] = { + "stored_aggregated_uncertainty": _float_or_none(row.get("aggregated_uncertainty")) + } + for col in _META_COLUMNS: + if col in df.columns: + meta[col] = _json_safe(row.get(col)) + store.row_meta[(stream, axis, bk)] = meta + return store + + @classmethod + def from_harvests(cls, harvests: dict[str, Any], *, round_idx: int = 1) -> "VoteStore": + """Populate round-1 votes directly from ``compute.harvest_pass`` outputs (T009). + + ``harvests`` maps pass label → ``PassHarvest`` (duck-typed: + ``presence_votes`` / ``identity_votes`` / ``utterance_votes`` bucket lists plus + ``quality_by_bucket`` / ``source_by_bucket``). This is the in-process + integration point for analyze_audio — no parquet round-trip; the parquet + ingest path (:meth:`from_run_dir`) remains for artifact-driven runs. + """ + store = cls() + for stream, harvest in harvests.items(): + for axis, buckets in ( + ("presence", harvest.presence_votes), + ("identity", harvest.identity_votes), + ("utterance", harvest.utterance_votes), + ): + for bucket in buckets: + bk = bucket_key(bucket["start"], bucket["end"]) + for source, payload in (bucket.get("votes") or {}).items(): + if not isinstance(payload, dict): + continue + store.add_vote( + Vote( + axis=axis, + bucket=bk, + source=str(source), + stream=stream, + scope="file", + round=round_idx, + payload=payload, + ) + ) + if axis == "presence": + meta: dict[str, Any] = {"stored_aggregated_uncertainty": None} + q = harvest.quality_by_bucket.get(bk) + s = harvest.source_by_bucket.get(bk) + if q: + meta.update({k: _json_safe(v) for k, v in q.items() if k in _META_COLUMNS}) + if s: + meta.update({k: _json_safe(v) for k, v in s.items() if k in _META_COLUMNS}) + store.row_meta[(stream, axis, bk)] = meta + else: + store.row_meta.setdefault((stream, axis, bk), {"stored_aggregated_uncertainty": None}) + return store + + # ── mutation ─────────────────────────────────────────────────────── + + def add_vote(self, vote: Vote) -> None: + """Insert/overwrite a vote; region scope shadows same (source, stream) file scope.""" + vid = vote.vote_id + self._votes[vid] = vote + key = (vote.stream, vote.axis, vote.bucket) + ids = self._index.setdefault(key, []) + if vid not in ids: + ids.append(vid) + self._round_added.setdefault(vote.round, []).append(vid) + if vote.scope.startswith("region:"): + for other_id in ids: + other = self._votes[other_id] + if other_id != vid and other.source == vote.source and other.scope == "file": + if other.status == "active": + other.status = "shadowed" + other.shadowed_by = vid + + def purge_source_in_bucket( + self, stream: str, bucket: tuple[float, float], source: str, *, reason: str, round_idx: int + ) -> int: + """Mark ``source``'s votes in ``bucket`` purged on presence + utterance axes (C10).""" + n = 0 + for axis in ("presence", "utterance"): + for vid in self._index.get((stream, axis, bucket), []): + v = self._votes[vid] + if v.source == source and v.status == "active": + v.status = "purged_hallucination" + v.provenance["purge_reason"] = reason + v.provenance["purge_round"] = round_idx + n += 1 + return n + + # ── reads ────────────────────────────────────────────────────────── + + def buckets(self, stream: str, axis: str) -> list[tuple[float, float]]: + """All known buckets for (stream, axis), time-ordered.""" + got = {bk for (s, a, bk) in self._index if s == stream and a == axis} + got |= {bk for (s, a, bk) in self.row_meta if s == stream and a == axis} + return sorted(got) + + def active_votes(self, stream: str, axis: str, bucket: tuple[float, float]) -> dict[str, dict[str, Any]]: + """Vote dict (source → payload) of active votes, as the aggregators expect.""" + out: dict[str, dict[str, Any]] = {} + for vid in self._index.get((stream, axis, bucket), []): + v = self._votes[vid] + if v.status == "active": + out[v.source] = v.payload + return out + + def votes_added_in_round(self, round_idx: int) -> list[Vote]: + """Votes first added in ``round_idx`` (for the append-only round files).""" + return [self._votes[vid] for vid in self._round_added.get(round_idx, [])] + + # ── aggregation (pure; FR-006) ───────────────────────────────────── + + def reaggregate_bucket( + self, stream: str, axis: str, bucket: tuple[float, float], *, aggregator: str + ) -> dict[str, Any]: + """Aggregate one bucket's active votes via the existing pure aggregators.""" + votes = self.active_votes(stream, axis, bucket) + p_voice: float | None = None + if axis == "presence": + agg = aggregate_presence(votes) + p_voice = presence_p_voice(votes) + elif axis == "identity": + agg = aggregate_identity(votes, raw_vs_enh=None, aggregator=aggregator) + else: + agg = aggregate_utterance(votes, aggregator=aggregator) + return { + "start": bucket[0], + "end": bucket[1], + "aggregated_uncertainty": agg, + "p_voice": p_voice, + "contributing_sources": sorted(votes.keys()), + } + + def parity_check(self, passes: list[str], *, aggregator: str, tol: float = 1e-9) -> dict[str, Any]: + """Re-aggregate every round-1 bucket and compare against the stored parquet values. + + This is the executable proof that aggregation is a pure function of the + vote store (tasks.md T007): a nonzero mismatch count means the split + missed an input. + + The comparison anchors on the **pre-coupling** scale: since FR-019 + (scene→utterance coupling, scene-quality-utterance US4) the parquet's + ``aggregated_uncertainty`` may carry a scene multiplier that is not a + function of the votes alone — the pure per-vote value is preserved on + ``raw_aggregated_uncertainty``, which is what the belief store computes + and compares (identical to ``aggregated_uncertainty`` on pre-FR-019 + artifacts and wherever coupling is 1.0). + """ + report: dict[str, Any] = {} + for stream in passes: + for axis in AXES: + n = mismatches = compared = 0 + max_abs = 0.0 + for bk in self.buckets(stream, axis): + n += 1 + meta = self.row_meta.get((stream, axis, bk)) or {} + stored = meta.get("raw_aggregated_uncertainty") + if stored is None or stored != stored: # NaN/missing → legacy column + stored = meta.get("stored_aggregated_uncertainty") + got = self.reaggregate_bucket(stream, axis, bk, aggregator=aggregator)["aggregated_uncertainty"] + if stored is None or got is None: + if (stored is None) != (got is None): + mismatches += 1 + continue + compared += 1 + diff = abs(float(stored) - float(got)) + max_abs = max(max_abs, diff) + if diff > tol: + mismatches += 1 + report[f"{stream}/{axis}"] = { + "buckets": n, + "compared": compared, + "mismatches": mismatches, + "max_abs_diff": max_abs, + } + return report + + +class BeliefState: + """Aggregated per-bucket state per (stream, axis), updated each round.""" + + def __init__(self, aggregator: str) -> None: + """Create an empty belief state using ``aggregator`` for identity/utterance.""" + self.aggregator = aggregator + self.rows: dict[tuple[str, str], list[dict[str, Any]]] = {} + + @classmethod + def from_store(cls, store: VoteStore, passes: list[str], *, aggregator: str) -> "BeliefState": + """Round-1 belief: aggregate every bucket, attach meta + epistemic/aleatoric split.""" + state = cls(aggregator) + for stream in passes: + for axis in AXES: + rows = [] + for bk in store.buckets(stream, axis): + row = store.reaggregate_bucket(stream, axis, bk, aggregator=aggregator) + meta = store.row_meta.get((stream, axis, bk)) or {} + row["meta"] = meta + _decompose(row, meta) + row["status"] = "open" + row["round"] = 1 + row["history"] = [{"round": 1, "aggregated_uncertainty": row["aggregated_uncertainty"]}] + rows.append(row) + state.rows[(stream, axis)] = rows + return state + + def update_buckets( + self, store: VoteStore, stream: str, axis: str, buckets: set[tuple[float, float]], round_idx: int + ) -> list[dict[str, Any]]: + """Incrementally re-aggregate only ``buckets`` (FR-006); returns changed rows.""" + changed = [] + for row in self.rows.get((stream, axis), []): + bk = bucket_key(row["start"], row["end"]) + if bk not in buckets: + continue + new = store.reaggregate_bucket(stream, axis, bk, aggregator=self.aggregator) + row["aggregated_uncertainty"] = new["aggregated_uncertainty"] + if new["p_voice"] is not None: + row["p_voice"] = new["p_voice"] + row["contributing_sources"] = new["contributing_sources"] + _decompose(row, row.get("meta") or {}) + row["round"] = round_idx + row["history"].append({"round": round_idx, "aggregated_uncertainty": row["aggregated_uncertainty"]}) + changed.append(row) + return changed + + def axis_rows(self, stream: str, axis: str) -> list[dict[str, Any]]: + """Rows for one (stream, axis), time-ordered.""" + return self.rows.get((stream, axis), []) + + def uncertainty_mass(self, stream: str, axis: str, theta_low: float) -> float: + """Σ max(0, u − θ_low) · width — the quantity interventions try to shrink.""" + total = 0.0 + for row in self.axis_rows(stream, axis): + u = row.get("aggregated_uncertainty") + if u is None: + continue + total += max(0.0, float(u) - theta_low) * (float(row["end"]) - float(row["start"])) + return total + + +def _decompose(row: dict[str, Any], meta: dict[str, Any]) -> None: + """Epistemic/aleatoric split (research.md D7). + + Floor = max(quality degradation, overlap posterior). The overlap term is + populated by I4 when segmentation-3.0 per-class posteriors are available; + otherwise the floor degrades to the quality-driven term only. + """ + agg = row.get("aggregated_uncertainty") + floor = 0.0 + for col in ("quality_snr", "quality_clip", "quality_reverb", "overlap_posterior"): + v = meta.get(col) + if v is not None: + try: + if v == v: # NaN guard + floor = max(floor, min(1.0, max(0.0, float(v)))) + except (TypeError, ValueError): + pass + row["aleatoric_floor"] = floor + row["epistemic"] = max(0.0, float(agg) - floor) if agg is not None else None + + +def _float_or_none(v: Any) -> float | None: # noqa: ANN401 + try: + f = float(v) + except (TypeError, ValueError): + return None + return None if f != f else f + + +def _json_safe(v: Any) -> Any: # noqa: ANN401 + """Coerce numpy scalars / NaN to plain JSON-safe python values.""" + if v is None or isinstance(v, (str, bool)): + return v + try: + import numpy as np + + if isinstance(v, np.generic): + v = v.item() + except ImportError: + pass + if isinstance(v, float) and v != v: + return None + return v diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/convergence.py b/src/senselab/audio/workflows/audio_analysis/adaptive/convergence.py new file mode 100644 index 000000000..4d790e19c --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/convergence.py @@ -0,0 +1,157 @@ +"""Convergence marking, round summaries, and the final report (FR-017/018/019).""" + +from __future__ import annotations + +from typing import Any + +from senselab.audio.workflows.audio_analysis.adaptive.belief import AXES, bucket_key + + +def apply_convergence_marks( + state: Any, # noqa: ANN401 — BeliefState + *, + passes: list[str], + policy: dict[str, Any], + touch_counts: dict[tuple[str, str, tuple[float, float]], int], + budget_left: bool, +) -> dict[str, int]: + """Update per-bucket status per FR-017; returns counts of status transitions. + + - ``converged``: uncertainty ≤ θ_low. + - ``irreducible``: touched ≥ max_region_rounds with < ε improvement AND the + aleatoric floor explains the residual (prototype floor = quality only; + reason ``snr_floor``) — or, without floor cover, marked + ``irreducible: no_reduction_under_available_interventions``. + - ``budget_exhausted``: interventions still wanted but the ledger is empty. + """ + th = policy["thresholds"] + theta_low, epsilon = float(th["theta_low"]), float(th["epsilon"]) + max_touch = int(policy["regions"]["max_region_rounds"]) + transitions = {"converged": 0, "irreducible": 0, "budget_exhausted": 0} + for stream in passes: + for axis in AXES: + for row in state.axis_rows(stream, axis): + if row.get("status") != "open": + continue + u = row.get("aggregated_uncertainty") + if u is None: + continue + if u <= theta_low: + row["status"] = "converged" + transitions["converged"] += 1 + continue + touches = touch_counts.get((stream, axis, bucket_key(row["start"], row["end"])), 0) + hist = row.get("history") or [] + improvement = None + if len(hist) >= 2: + improvement = hist[-2]["aggregated_uncertainty"] - hist[-1]["aggregated_uncertainty"] + stalled = improvement is not None and improvement < epsilon + if touches >= max_touch and stalled: + floor = float(row.get("aleatoric_floor") or 0.0) + if u <= floor + epsilon: + row["status"] = "irreducible" + row["irreducible_reason"] = "snr_floor" + else: + row["status"] = "irreducible" + row["irreducible_reason"] = "no_reduction_under_available_interventions" + transitions["irreducible"] += 1 + elif touches >= 1 and not budget_left: + row["status"] = "budget_exhausted" + transitions["budget_exhausted"] += 1 + return transitions + + +def round_summary( + *, + round_idx: int, + state: Any, # noqa: ANN401 + passes: list[str], + policy: dict[str, Any], + fired: list[dict[str, Any]], + not_admitted: list[dict[str, Any]], + mass_before: dict[str, float], + ledger: Any, # noqa: ANN401 +) -> dict[str, Any]: + """One ``rounds//summary.json`` payload.""" + theta_low = float(policy["thresholds"]["theta_low"]) + mass_after = {f"{s}/{a}": round(state.uncertainty_mass(s, a, theta_low), 6) for s in passes for a in AXES} + statuses: dict[str, int] = {} + for s in passes: + for a in AXES: + for row in state.axis_rows(s, a): + statuses[row.get("status", "open")] = statuses.get(row.get("status", "open"), 0) + 1 + return { + "round": round_idx, + "interventions": { + "fired": [c["intervention_id"] for c in fired], + "deferred_budget": [c["rule"] for c in not_admitted if c["status"] == "deferred_budget"], + "blocked_guard": [ + {"rule": c["rule"], "reason": c.get("error")} for c in not_admitted if c["status"] == "blocked_guard" + ], + "failed": [c["intervention_id"] for c in fired if c.get("exec_status") == "failed"], + }, + "budget": ledger.as_dict(), + "uncertainty_mass": {"before": mass_before, "after": mass_after}, + "bucket_statuses": statuses, + } + + +def build_convergence_report( + *, + state: Any, # noqa: ANN401 + passes: list[str], + policy: dict[str, Any], + rounds: list[dict[str, Any]], + ledger: Any, # noqa: ANN401 + iterations: list[dict[str, Any]], + run_state: str, + provenance: dict[str, Any], +) -> dict[str, Any]: + """``final/convergence.json`` per data-model.md ConvergenceReport.""" + per_axis: dict[str, Any] = {} + irreducible_regions: list[dict[str, Any]] = [] + for stream in passes: + for axis in AXES: + rows = state.axis_rows(stream, axis) + counts: dict[str, int] = {} + for row in rows: + counts[row.get("status", "open")] = counts.get(row.get("status", "open"), 0) + 1 + if row.get("status") == "irreducible": + irreducible_regions.append( + { + "axis": axis, + "stream": stream, + "start": row["start"], + "end": row["end"], + "reason": row.get("irreducible_reason"), + "residual": row.get("aggregated_uncertainty"), + "floor": row.get("aleatoric_floor"), + } + ) + per_axis[f"{stream}/{axis}"] = { + "buckets": len(rows), + **counts, + "residual_mass": round( + state.uncertainty_mass(stream, axis, float(policy["thresholds"]["theta_low"])), 6 + ), + } + next_actions = [ + { + "rule": e["rule"], + "region_id": e.get("region_id"), + "priority": e.get("priority"), + "status": e["status"], + "reason": e.get("error"), + } + for e in iterations + if e["status"] in ("deferred_budget", "blocked_guard") + ] + return { + "run_state": run_state, + "rounds": rounds, + "per_axis": per_axis, + "irreducible_regions": irreducible_regions, + "budget": ledger.as_dict(), + "next_actions": next_actions, + **provenance, + } diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/evaluate.py b/src/senselab/audio/workflows/audio_analysis/adaptive/evaluate.py new file mode 100644 index 000000000..f65a12e58 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/evaluate.py @@ -0,0 +1,216 @@ +"""Ground-truth evaluation against a Label Studio export (tasks.md T034). + +Consumes the LS JSON export format (list of tasks, ``annotations[].result`` +with paired ``labels``/``textarea`` items sharing region ids) and scores: + +- **presence**: bucket-level accuracy/precision/recall of ``p_voice ≥ 0.5`` + against labeled speech spans; mean uncertainty inside vs outside speech. +- **transcript**: WER of the fused consensus (and each contributing model) + against the concatenated GT texts, computed only over words whose midpoint + falls inside a *transcribed* GT segment (untranscribed GT spans are excluded + from both sides — the annotator's own uncertainty is not a reference). +- **diarization**: greedy cluster↔GT-speaker mapping by time overlap + + speaker-attribution accuracy over fused words; speaker-count comparison. +- **boundary/uncertainty checks**: identity uncertainty at GT speaker + boundaries vs within segments; utterance uncertainty + fused word confidence + inside the untranscribed GT span vs elsewhere (the region a human could not + transcribe should be where the pipeline is least certain). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from senselab.audio.workflows.audio_analysis.aggregate import _normalize_transcript_for_wer +from senselab.audio.workflows.audio_analysis.harvesters import _levenshtein + +_TOKEN_EQUIV = {"u": "you"} # annotator shorthand normalization, reported separately + + +# LS export *parsing* lives next to the export builders (architecture-review +# T049); re-exported here for existing callers. +from senselab.audio.workflows.audio_analysis.labelstudio import ( # noqa: E402, F401 + load_ls_ground_truth, +) + + +def _tokens(text: str, *, equiv: bool) -> list[str]: + toks = _normalize_transcript_for_wer(text or "").split() + return [_TOKEN_EQUIV.get(t, t) for t in toks] if equiv else toks + + +def _wer(ref: list[str], hyp: list[str]) -> float | None: + """WER over token lists — jiwer-backed when available (T049), Levenshtein fallback.""" + if not ref: + return None + try: + from senselab.audio.tasks.speech_to_text_evaluation import calculate_wer # noqa: PLC0415 + + return round(float(calculate_wer(" ".join(ref), " ".join(hyp))), 4) + except (ImportError, ModuleNotFoundError): + return round(_levenshtein(ref, hyp) / len(ref), 4) + + +def _in_any(mid: float, spans: list[tuple[float, float]]) -> bool: + return any(s <= mid < e for s, e in spans) + + +def evaluate_against_ground_truth( + *, + out_dir: Path, + gt_path: Path, + word_streams: dict[str, list[dict[str, Any]]] | None = None, +) -> dict[str, Any]: + """Score final outputs in ``out_dir`` against the LS ground truth; write eval.json.""" + gt = load_ls_ground_truth(gt_path) + final = Path(out_dir) / "final" + transcript = json.loads((final / "transcript.json").read_text()) + diarization = json.loads((final / "diarization.json").read_text()) + import pandas as pd + + presence = pd.read_parquet(final / "presence.parquet") + + speech_spans = [(s["start"], s["end"]) for s in gt["segments"]] + transcribed = [(s["start"], s["end"]) for s in gt["segments"] if s["text"]] + untranscribed = [(s["start"], s["end"]) for s in gt["segments"] if not s["text"]] + + # ── presence ───────────────────────────────────────────────────────── + tp = fp = fn = tn = 0 + unc_speech: list[float] = [] + unc_sil: list[float] = [] + for _, row in presence.iterrows(): + mid = (float(row["start"]) + float(row["end"])) / 2.0 + pv = row.get("p_voice") + if pv is None or pv != pv: + continue + gt_speech = _in_any(mid, speech_spans) + pred = float(pv) >= 0.5 + tp += gt_speech and pred + fp += (not gt_speech) and pred + fn += gt_speech and (not pred) + tn += (not gt_speech) and (not pred) + u = row.get("aggregated_uncertainty") + if u is not None and u == u: + (unc_speech if gt_speech else unc_sil).append(float(u)) + presence_eval = { + "buckets": tp + fp + fn + tn, + "accuracy": round((tp + tn) / max(1, tp + fp + fn + tn), 4), + "precision": round(tp / max(1, tp + fp), 4), + "recall": round(tp / max(1, tp + fn), 4), + "mean_uncertainty_in_gt_speech": round(sum(unc_speech) / len(unc_speech), 4) if unc_speech else None, + "mean_uncertainty_in_gt_silence": round(sum(unc_sil) / len(unc_sil), 4) if unc_sil else None, + } + + # ── transcript (fused + per model) ────────────────────────────────── + ref_tokens = [t for s in gt["segments"] if s["text"] for t in _tokens(s["text"], equiv=True)] + + def _score_words(words: list[dict[str, Any]]) -> dict[str, Any]: + hyp_words = [w for w in words if _in_any((w["start"] + w["end"]) / 2.0, transcribed)] + hyp_plain = [t for w in hyp_words for t in _tokens(w["text"], equiv=False)] + hyp_equiv = [t for w in hyp_words for t in _tokens(w["text"], equiv=True)] + ref_plain = [t for s in gt["segments"] if s["text"] for t in _tokens(s["text"], equiv=False)] + return { + "n_words_scored": len(hyp_equiv), + "wer": _wer(ref_plain, hyp_plain), + "wer_normalized": _wer(ref_tokens, hyp_equiv), + } + + transcript_eval: dict[str, Any] = {"fused": _score_words(transcript["words"])} + if word_streams: + transcript_eval["per_model"] = {m: _score_words(ws) for m, ws in sorted(word_streams.items())} + + # ── diarization ────────────────────────────────────────────────────── + overlap: dict[tuple[str, str], float] = {} + for seg in diarization.get("segments") or []: + for g in gt["segments"]: + ov = min(seg["end"], g["end"]) - max(seg["start"], g["start"]) + if ov > 0 and g["speaker"]: + key = (seg["cluster_id"], g["speaker"]) + overlap[key] = overlap.get(key, 0.0) + ov + mapping: dict[str, str] = {} + used_gt: set[str] = set() + for (cid, spk), _ov in sorted(overlap.items(), key=lambda kv: -kv[1]): + if cid not in mapping and spk not in used_gt: + mapping[cid] = spk + used_gt.add(spk) + attributed = correct = 0 + for w in transcript["words"]: + mid = (w["start"] + w["end"]) / 2.0 + gt_spk = next((s["speaker"] for s in gt["segments"] if s["start"] <= mid < s["end"]), None) + if gt_spk is None or w.get("speaker") is None: + continue + attributed += 1 + correct += mapping.get(w["speaker"]) == gt_spk + # Boundary F1: predicted segment starts vs GT starts (±tol), excluding t=0. + tol = 0.25 + gt_bounds = [g["start"] for g in gt["segments"][1:]] + pred_bounds = sorted({round(s["start"], 3) for s in (diarization.get("segments") or [])[1:]}) + b_tp = sum(1 for b in gt_bounds if any(abs(b - p) <= tol for p in pred_bounds)) + b_prec = b_tp / len(pred_bounds) if pred_bounds else None + b_rec = b_tp / len(gt_bounds) if gt_bounds else None + b_f1 = ( + round(2 * b_prec * b_rec / (b_prec + b_rec), 4) + if b_prec and b_rec and (b_prec + b_rec) > 0 + else (0.0 if pred_bounds or gt_bounds else None) + ) + diarization_eval = { + "gt_speakers": sorted({s["speaker"] for s in gt["segments"] if s["speaker"]}), + "predicted_clusters": [c["cluster_id"] for c in diarization.get("clusters") or []], + "refined": diarization.get("refined", False), + "cluster_to_gt": mapping, + "word_speaker_accuracy": round(correct / attributed, 4) if attributed else None, + "n_words_attributed": attributed, + "boundary_f1": b_f1, + "boundary_precision": round(b_prec, 4) if b_prec is not None else None, + "boundary_recall": round(b_rec, 4) if b_rec is not None else None, + "n_pred_speakers": len(diarization.get("clusters") or []), + "n_gt_speakers": len({s["speaker"] for s in gt["segments"] if s["speaker"]}), + } + + # ── uncertainty localization checks ────────────────────────────────── + def _mean_conf(spans: list[tuple[float, float]]) -> float | None: + vals = [w["confidence"] for w in transcript["words"] if _in_any((w["start"] + w["end"]) / 2.0, spans)] + return round(sum(vals) / len(vals), 4) if vals else None + + rows2 = json.loads((Path(out_dir) / "rounds").joinpath("1", "summary.json").read_text()) + localization = { + "untranscribed_gt_spans": untranscribed, + "fused_confidence_in_untranscribed": _mean_conf(untranscribed), + "fused_confidence_in_transcribed": _mean_conf(transcribed), + "round1_uncertainty_mass": rows2.get("uncertainty_mass"), + } + # Identity uncertainty at GT speaker boundaries vs inside segments. + try: + import pandas as pd # noqa: PLC0415 + + last_round = max(int(p.name) for p in (Path(out_dir) / "rounds").iterdir() if p.name.isdigit()) + ident = pd.read_parquet(Path(out_dir) / "rounds" / str(last_round) / "belief" / "identity.parquet") + ident = ident[ident["stream"] == transcript.get("stream", "raw_16k")] + boundaries = [g["start"] for g in gt["segments"][1:]] + at_b: list[float] = [] + inside: list[float] = [] + for _, row in ident.iterrows(): + u = row.get("aggregated_uncertainty") + if u is None or u != u: + continue + if any(row["start"] <= b < row["end"] for b in boundaries): + at_b.append(float(u)) + elif _in_any((row["start"] + row["end"]) / 2.0, speech_spans): + inside.append(float(u)) + localization["identity_uncertainty_at_gt_boundaries"] = round(sum(at_b) / len(at_b), 4) if at_b else None + localization["identity_uncertainty_within_segments"] = round(sum(inside) / len(inside), 4) if inside else None + except (OSError, ValueError): + pass + + eval_doc = { + "ground_truth": str(gt_path), + "gt_segments": gt["segments"], + "presence": presence_eval, + "transcript": transcript_eval, + "diarization": diarization_eval, + "localization": localization, + } + (final / "eval.json").write_text(json.dumps(eval_doc, indent=2, default=str)) + return eval_doc diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/fusion.py b/src/senselab/audio/workflows/audio_analysis/adaptive/fusion.py new file mode 100644 index 000000000..d88ad1179 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/fusion.py @@ -0,0 +1,301 @@ +"""Fusion: time-aligned word-slot voting → final outputs (FR-021/022, research.md D9). + +Post-T050 the fusion *math* lives in the reusable task package +``senselab.audio.tasks.speech_to_text_ensemble`` (``fuse_word_streams`` / +``load_calibrator`` / ``iter_word_leaves`` are re-exported here for the loop's +callers); this module keeps the workflow-specific parts — artifact word-stream +collection, policy → weights/params translation, speaker & presence lookups +from the belief state, and the ``final/`` artifact writers. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from senselab.audio.tasks.speech_to_text_ensemble import ( # noqa: F401 — re-exported for loop callers + fuse_word_streams, + iter_word_leaves, + load_calibrator, +) +from senselab.audio.workflows.audio_analysis.adaptive.belief import bucket_key +from senselab.audio.workflows.audio_analysis.adaptive.policy import family_weights + +# ── word-stream extraction ─────────────────────────────────────────────── + + +def collect_word_streams( + asr_by_model: dict[str, dict[str, Any]], + align_by_model: dict[str, dict[str, Any]], + *, + purged_spans: list[tuple[float, float, str]] | None = None, +) -> dict[str, list[dict[str, Any]]]: + """Per-model timestamped word lists; alignment result wins for text-only models. + + ``purged_spans`` = (start, end, model) spans adjudicated as hallucinations — + those words are excluded from fusion (C10 consequence). + """ + streams: dict[str, list[dict[str, Any]]] = {} + for model, block in asr_by_model.items(): + source = block + align = align_by_model.get(model) + words = iter_word_leaves((source.get("result") if isinstance(source, dict) else None) or []) + has_ts = bool(words) + if (not has_ts) and isinstance(align, dict) and align.get("status") == "ok": + words = iter_word_leaves(align.get("result") or []) + if not words: + continue + words.sort(key=lambda w: (w["start"], w["end"])) + if purged_spans: + words = [ + w + for w in words + if not any(m == model and w["start"] < e and w["end"] > s for (s, e, m) in purged_spans) + ] + streams[model] = words + return streams + + +# ── slot voting ────────────────────────────────────────────────────────── + + +def fuse_words( + word_streams: dict[str, list[dict[str, Any]]], + *, + policy: dict[str, Any], + speaker_at: Any = None, # noqa: ANN401 — callable (t) -> str | None + p_voice_at: Any = None, # noqa: ANN401 — callable (t) -> float | None + calibrator: Any = None, # noqa: ANN401 — callable (c) -> c' | None +) -> list[dict[str, Any]]: + """Policy-driven wrapper over the reusable transcript-ensemble task (T050). + + Translates the adaptive policy into the task API's explicit arguments — + model-family weights (FR-008) and the fusion slot/margin parameters — and + delegates the voting math to + :func:`senselab.audio.tasks.speech_to_text_ensemble.fuse_word_streams`. + """ + fus = policy["fusion"] + return fuse_word_streams( + word_streams, + weights=family_weights(sorted(word_streams), policy), + slot_overlap=float(fus["slot_overlap"]), + slot_mid_tol_s=float(fus["slot_mid_tol_s"]), + winner_margin=float(fus["winner_margin"]), + alternate_min_share=float(fus["alternate_min_share"]), + speaker_at=speaker_at, + p_voice_at=p_voice_at, + calibrator=calibrator, + ) + + +# ── lookups from belief state ──────────────────────────────────────────── + + +def make_speaker_lookup(store: Any, state: Any, stream: str) -> Any: # noqa: ANN401 + """(t) → majority unified cluster_id across active diarization votes at t.""" + rows = state.axis_rows(stream, "identity") + + def lookup(t: float) -> str | None: + for row in rows: + if row["start"] <= t < row["end"]: + bk = bucket_key(row["start"], row["end"]) + counts: dict[str, int] = {} + for source, payload in store.active_votes(stream, "identity", bk).items(): + if source.startswith("__") or "::" in source: + continue + cid = payload.get("cluster_id") + if cid and cid not in ("SIL", ""): + counts[str(cid)] = counts.get(str(cid), 0) + 1 + if not counts: + return None + return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] + return None + + return lookup + + +def make_p_voice_lookup(state: Any, stream: str) -> Any: # noqa: ANN401 + """(t) → presence p_voice at t from the presence belief rows.""" + rows = state.axis_rows(stream, "presence") + + def lookup(t: float) -> float | None: + best = None + for row in rows: + if row["start"] <= t < row["end"]: + pv = row.get("p_voice") + if pv is not None: + best = pv if best is None else max(best, pv) + return best + + return lookup + + +# ── final artifact writers ─────────────────────────────────────────────── + + +def build_final_outputs( + *, + out_dir: Path, + words: list[dict[str, Any]], + store: Any, # noqa: ANN401 + state: Any, # noqa: ANN401 + stream: str, + policy: dict[str, Any], + generated_from_round: int, + refined_identity: dict[str, Any] | None = None, + calibrated: bool = False, + timestamps_meta: dict[str, Any] | None = None, + language: str | None = None, +) -> dict[str, Any]: + """Write final/{transcript,diarization}.json (+.rttm) + final/presence.parquet; return transcript doc.""" + final = out_dir / "final" + final.mkdir(parents=True, exist_ok=True) + + base_speaker_lookup = make_speaker_lookup(store, state, stream) + if refined_identity is not None: + from senselab.audio.workflows.audio_analysis.adaptive.identity_repair import cluster_at + + def speaker_lookup(t: float) -> str | None: + return cluster_at(refined_identity, t) or base_speaker_lookup(t) + else: + speaker_lookup = base_speaker_lookup + p_voice_lookup = make_p_voice_lookup(state, stream) + + # transcript.json — segments rollup on speaker change or >0.5 s word gap. + segments: list[dict[str, Any]] = [] + for w in words: + if segments and w.get("speaker") == segments[-1]["speaker"] and w["start"] - segments[-1]["end"] <= 0.5: + seg = segments[-1] + seg["end"] = w["end"] + seg["text"] += " " + w["text"] + seg["min_word_confidence"] = min(seg["min_word_confidence"], w["confidence"]) + else: + segments.append( + { + "start": w["start"], + "end": w["end"], + "speaker": w.get("speaker"), + "text": w["text"], + "min_word_confidence": w["confidence"], + } + ) + transcript = { + "calibrated": calibrated, + "policy_hash": policy.get("policy_hash"), + "generated_from_round": generated_from_round, + "stream": stream, + "language": language, + "timestamps": timestamps_meta or {"timestamps_source": "member_vote"}, + "words": words, + "segments": segments, + } + (final / "transcript.json").write_text(json.dumps(transcript, indent=2)) + + # diarization.json — refined I2 segments when available (real boundary + # confidences from change-point prominence); else merge identity buckets + # by majority cluster where voiced. + diar_segments: list[dict[str, Any]] = [] + clusters: dict[str, dict[str, Any]] = {} + if refined_identity is not None: + for seg in refined_identity["segments"]: + diar_segments.append( + { + "start": seg["start"], + "end": seg["end"], + "cluster_id": seg["cluster_id"], + "boundary_confidence": seg.get("boundary_confidence", {"start": 0.5, "end": 0.5}), + } + ) + c = clusters.setdefault( + seg["cluster_id"], {"cluster_id": seg["cluster_id"], "total_speech_s": 0.0, "n_segments": 0} + ) + c["total_speech_s"] = round(c["total_speech_s"] + (seg["end"] - seg["start"]), 6) + c["n_segments"] += 1 + else: + for row in state.axis_rows(stream, "identity"): + mid = (row["start"] + row["end"]) / 2.0 + cid = speaker_lookup(mid) + pv = p_voice_lookup(mid) + if cid is None or (pv is not None and pv < 0.5): + continue + if ( + diar_segments + and diar_segments[-1]["cluster_id"] == cid + and row["start"] <= diar_segments[-1]["end"] + 1e-6 + ): + diar_segments[-1]["end"] = row["end"] + else: + diar_segments.append( + { + "start": row["start"], + "end": row["end"], + "cluster_id": cid, + "boundary_confidence": {"start": 0.5, "end": 0.5}, + } + ) + c = clusters.setdefault(cid, {"cluster_id": cid, "total_speech_s": 0.0, "n_segments": 0}) + c["total_speech_s"] = round(c["total_speech_s"] + (row["end"] - row["start"]), 6) + for seg in diar_segments: + clusters[seg["cluster_id"]]["n_segments"] += 1 + # contracts/final-outputs.md: member_labels (refined cluster ↔ diar-model raw + # labels via vote co-occurrence) + per-segment overlap flag (I4 posterior). + identity_rows = state.axis_rows(stream, "identity") + member_labels: dict[str, dict[str, set]] = {} + for seg in diar_segments: + labels_for_cluster = member_labels.setdefault(str(seg["cluster_id"]), {}) + for row in identity_rows: + mid = (row["start"] + row["end"]) / 2.0 + if not (seg["start"] <= mid < seg["end"]): + continue + bk = bucket_key(row["start"], row["end"]) + for source, payload in store.active_votes(stream, "identity", bk).items(): + if source.startswith(("__", "embedding_")) or "::" in source: + continue + raw_label = payload.get("speaker_label") + if raw_label and raw_label not in ("SIL", ""): + labels_for_cluster.setdefault(source, set()).add(str(raw_label)) + seg["overlap"] = any( + (row.get("overlap_posterior") or 0.0) >= 0.5 + for row in identity_rows + if seg["start"] <= (row["start"] + row["end"]) / 2.0 < seg["end"] + ) + for cluster in clusters.values(): + cluster["member_labels"] = { + model: sorted(labels) for model, labels in (member_labels.get(str(cluster["cluster_id"])) or {}).items() + } + diarization = { + "clusters": sorted(clusters.values(), key=lambda c: c["cluster_id"]), + "segments": diar_segments, + "refined": refined_identity is not None, + } + (final / "diarization.json").write_text(json.dumps(diarization, indent=2)) + + # RTTM sidecar for interop (contracts/final-outputs.md). + audio_id = policy.get("rttm_audio_id") or "audio" + rttm_lines = [ + f"SPEAKER {audio_id} 1 {seg['start']:.3f} {max(0.0, seg['end'] - seg['start']):.3f} " + f" {seg['cluster_id']} " + for seg in diar_segments + ] + (final / "diarization.rttm").write_text("\n".join(rttm_lines) + ("\n" if rttm_lines else "")) + + # presence.parquet — final presence belief. + import pandas as pd + + pres_rows = [ + { + "start": r["start"], + "end": r["end"], + "p_voice": r.get("p_voice"), + "aggregated_uncertainty": r.get("aggregated_uncertainty"), + "epistemic": r.get("epistemic"), + "aleatoric_floor": r.get("aleatoric_floor"), + "status": r.get("status"), + "irreducible_reason": r.get("irreducible_reason"), + "round": r.get("round"), + } + for r in state.axis_rows(stream, "presence") + ] + pd.DataFrame(pres_rows).to_parquet(final / "presence.parquet", index=False) + return transcript diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/identity_repair.py b/src/senselab/audio/workflows/audio_analysis/adaptive/identity_repair.py new file mode 100644 index 000000000..367dbed79 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/identity_repair.py @@ -0,0 +1,252 @@ +"""Identity repair: embedding change-point detection + consensus re-clustering. + +Implements I1 (boundary evidence) and I2 (re-cluster) generically: + +1. Per embedding model, L2-normalize the per-window vectors and compute the + adjacent-window cosine-distance trajectory; average trajectories across + models (windows share one grid per run). +2. Change-points = local maxima above ``mean + k·std`` (policy ``cp_k``) with a + minimum absolute floor (``cp_floor``); prominence is kept as boundary + confidence. Diarization-model boundaries join the candidate cut set — the + diarizers may be *under*-segmenting (the failure this repairs) but any cut + they did make is evidence. +3. Segments = voiced spans cut at change-points (min duration ``min_segment_s``; + shorter segments merge into the neighbor with the closer centroid). +4. Per model, pool window embeddings per segment (p_voice-weighted mean) and + agglomerative-cluster the pooled vectors (average linkage, cosine threshold + ``recluster_cosine_threshold``). Cross-model consensus: segments co-clustered + by ≥ half the models merge (union-find on the co-association matrix). +5. Output refined segments/clusters (ids ``R0, R1, …`` by first appearance), + boundary confidences, and per-bucket identity votes; the caller shadows the + per-bucket ``__cross_diar_label_disagreement__`` with a recomputed value that + includes the new voter. + +No parameter here is tuned to a particular file: everything comes from the +policy, and inputs are whatever embeddings/diarization the run produced. +""" + +from __future__ import annotations + +from typing import Any + + +def _l2(mat: Any) -> Any: # noqa: ANN401 + import numpy as np + + norms = np.linalg.norm(mat, axis=1, keepdims=True) + return mat / np.maximum(norms, 1e-9) + + +def change_point_trajectory( + window_embeddings: dict[str, list[dict[str, Any]]], +) -> tuple[list[float], list[float]]: + """Mean adjacent-cosine-distance trajectory across models → (boundary_times, distances).""" + import numpy as np + + per_model: list[Any] = [] + times: list[float] | None = None + for windows in window_embeddings.values(): + if len(windows) < 2: + continue + vecs = _l2(np.asarray([w["vector"] for w in windows], dtype=float)) + d = 1.0 - (vecs[:-1] * vecs[1:]).sum(axis=1) + per_model.append(d) + if times is None: + # Boundary between window i and i+1 ≈ centre of their overlap. + times = [ + round((float(windows[i + 1]["start_s"]) + float(windows[i]["end_s"])) / 2.0, 4) + for i in range(len(windows) - 1) + ] + if not per_model or times is None: + return [], [] + dist = np.mean(np.stack(per_model), axis=0) + if len(dist) >= 3: # light smoothing, preserves peaks + dist = np.convolve(dist, np.array([0.25, 0.5, 0.25]), mode="same") + return times, [float(x) for x in dist] + + +def detect_change_points( + times: list[float], dist: list[float], *, cp_k: float, cp_floor: float +) -> list[dict[str, Any]]: + """Local maxima above max(mean + k·std, floor); prominence-normalized confidence.""" + import numpy as np + + if not times: + return [] + d = np.asarray(dist) + thr = max(float(d.mean() + cp_k * d.std()), cp_floor) + span = max(1e-9, float(d.max() - d.min())) + out = [] + for i in range(len(d)): + left = d[i - 1] if i > 0 else -np.inf + right = d[i + 1] if i < len(d) - 1 else -np.inf + if d[i] >= thr and d[i] >= left and d[i] >= right: + conf = round(float((d[i] - d.min()) / span), 4) + out.append({"time": times[i], "distance": round(float(d[i]), 4), "confidence": conf}) + return out + + +def _voiced_spans( + p_voice_at: Any, # noqa: ANN401 — callable (t) -> float | None + duration_s: float, + *, + step: float = 0.05, + threshold: float = 0.5, +) -> list[tuple[float, float]]: + spans: list[tuple[float, float]] = [] + t, open_start = 0.0, None + while t < duration_s: + pv = p_voice_at(t + step / 2) + voiced = pv is not None and pv >= threshold + if voiced and open_start is None: + open_start = t + elif not voiced and open_start is not None: + spans.append((round(open_start, 4), round(t, 4))) + open_start = None + t += step + if open_start is not None: + spans.append((round(open_start, 4), round(duration_s, 4))) + return spans + + +def _agglomerative_cosine(vectors: Any, threshold: float) -> list[int]: # noqa: ANN401 + """Deterministic average-linkage agglomerative clustering on cosine distance.""" + import numpy as np + + n = len(vectors) + clusters: list[list[int]] = [[i] for i in range(n)] + vecs = _l2(np.asarray(vectors, dtype=float)) + dmat = 1.0 - vecs @ vecs.T + while len(clusters) > 1: + best: tuple[float, int, int] | None = None + for a in range(len(clusters)): + for b in range(a + 1, len(clusters)): + d = float(np.mean([dmat[i, j] for i in clusters[a] for j in clusters[b]])) + if best is None or d < best[0] - 1e-12: + best = (d, a, b) + if best is None or best[0] > threshold: + break + _, a, b = best + clusters[a] = clusters[a] + clusters[b] + del clusters[b] + labels = [0] * n + for lbl, members in enumerate(sorted(clusters, key=min)): + for i in members: + labels[i] = lbl + return labels + + +def repair_identity( + *, + window_embeddings: dict[str, list[dict[str, Any]]], + diar_boundaries: list[float], + p_voice_at: Any, # noqa: ANN401 — callable (t) -> float | None + duration_s: float, + policy: dict[str, Any], +) -> dict[str, Any] | None: + """Full I1+I2 repair. Returns refined segments/clusters + change-point evidence, or None.""" + import numpy as np + + cfg = policy.get("identity") or {} + times, dist = change_point_trajectory(window_embeddings) + if not times: + return None + cps = detect_change_points(times, dist, cp_k=float(cfg.get("cp_k", 1.0)), cp_floor=float(cfg.get("cp_floor", 0.15))) + cuts = sorted({round(c["time"], 4) for c in cps} | {round(b, 4) for b in diar_boundaries}) + min_seg = float(cfg.get("min_segment_s", 0.25)) + + # Voiced spans cut at change-points. + segments: list[dict[str, Any]] = [] + voiced_thr = float(cfg.get("voiced_threshold", 0.5)) + for span_start, span_end in _voiced_spans(p_voice_at, duration_s, threshold=voiced_thr): + edges = [span_start] + [c for c in cuts if span_start + min_seg <= c <= span_end - min_seg] + [span_end] + for i in range(len(edges) - 1): + if edges[i + 1] - edges[i] >= min_seg: + segments.append({"start": edges[i], "end": edges[i + 1]}) + if not segments: + return None + + # Pool per model per segment (p_voice-weighted window means). + pooled: dict[str, list[Any]] = {} + for model, windows in window_embeddings.items(): + vecs = _l2(np.asarray([w["vector"] for w in windows], dtype=float)) + mids = np.asarray([(float(w["start_s"]) + float(w["end_s"])) / 2.0 for w in windows]) + seg_vecs = [] + for seg in segments: + inside = (mids >= seg["start"]) & (mids < seg["end"]) + if not inside.any(): # fall back to nearest window + inside = np.zeros(len(mids), dtype=bool) + inside[int(np.argmin(np.abs(mids - (seg["start"] + seg["end"]) / 2)))] = True + weights = np.asarray([max(0.05, p_voice_at(m) or 0.05) for m in mids[inside]]) + v = (vecs[inside] * weights[:, None]).sum(axis=0) / weights.sum() + seg_vecs.append(v) + pooled[model] = seg_vecs + + # Per-model clustering → cross-model co-association consensus. + thr = float(cfg.get("recluster_cosine_threshold", 0.45)) + n = len(segments) + coassoc = np.zeros((n, n)) + for model, seg_vecs in pooled.items(): + labels = _agglomerative_cosine(seg_vecs, thr) + for i in range(n): + for j in range(n): + coassoc[i, j] += labels[i] == labels[j] + coassoc /= max(1, len(pooled)) + + parent = list(range(n)) + + def find(x: int) -> int: + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + for i in range(n): + for j in range(i + 1, n): + if coassoc[i, j] >= 0.5: + parent[find(j)] = find(i) + roots: dict[int, str] = {} + for idx, seg in enumerate(segments): + r = find(idx) + if r not in roots: + roots[r] = f"R{len(roots)}" + seg["cluster_id"] = roots[r] + cp_conf = {round(c["time"], 4): c["confidence"] for c in cps} + seg["boundary_confidence"] = { + "start": cp_conf.get(round(seg["start"], 4), 0.5), + "end": cp_conf.get(round(seg["end"], 4), 0.5), + } + return { + "segments": segments, + "n_clusters": len(roots), + "change_points": cps, + "trajectory": {"times": times, "distances": [round(d, 4) for d in dist]}, + "models_used": sorted(window_embeddings.keys()), + "params": { + "cp_k": cfg.get("cp_k", 1.0), + "cp_floor": cfg.get("cp_floor", 0.15), + "min_segment_s": min_seg, + "recluster_cosine_threshold": thr, + }, + } + + +def cluster_at(refined: dict[str, Any], t: float) -> str | None: + """Refined cluster id at time ``t`` (None in unvoiced gaps).""" + for seg in refined["segments"]: + if seg["start"] <= t < seg["end"]: + return str(seg["cluster_id"]) + return None + + +def cross_source_disagreement(cluster_ids: list[str]) -> float | None: + """Fraction of source pairs disagreeing on the cluster — mirrors the identity axis sub-signal.""" + ids = [c for c in cluster_ids if c] + if len(ids) < 2: + return None + pairs = disagree = 0 + for i in range(len(ids)): + for j in range(i + 1, len(ids)): + pairs += 1 + disagree += ids[i] != ids[j] + return disagree / pairs if pairs else None diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/interventions.py b/src/senselab/audio/workflows/audio_analysis/adaptive/interventions.py new file mode 100644 index 000000000..57e45b041 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/interventions.py @@ -0,0 +1,981 @@ +"""Intervention catalog (prototype) — contracts/interventions.md. + +Implemented for real on artifacts + the content-addressable cache: + +- ``S1_stream_election`` — per-region raw/enhanced election from belief evidence. +- ``P3_hallucination_adjudication`` / ``C9_missed_speech`` — cross-signal repair + over existing evidence (C10 / C9), degraded to the indicators present in the + ingested run (no whisper ``no_speech_prob`` / PPG unless the run had them). +- ``U2_reserve_escalation`` — adds reserve ASR models by **cache replay**: the + reserve model's whole-file result is read from ``analyze_audio``'s + content-addressable cache (same audio signature ⇒ same waveform), windowed to + the region's buckets with the *same* harvest math the comparator uses + (``harvest_utterance_votes``), and merged as region-scoped votes. + +- ``U1_region_reasr`` — live region re-ASR (HF whisper pipeline pool, enhanced stream + regenerated on demand with recorded raw fallback). +- ``I1_boundary_refinement`` / ``I2_recluster`` — identity repair from per-window + embeddings (stored artifacts, or live fine-hop via ``backends.embed_windows``). +- ``I4_overlap_detection`` — segmentation-3.0 per-class posteriors (gated model; + guards to ``next_actions`` without a token). + +Still deferred: ``P2_fine_posteriors`` and v2's ``U4_overlap_separation`` +(contracts/interventions.md). +""" + +from __future__ import annotations + +import importlib.util +import json +import math +from pathlib import Path +from typing import Any + +from senselab.audio.workflows.audio_analysis.adaptive.belief import Vote, bucket_key +from senselab.audio.workflows.audio_analysis.adaptive.policy import family_weights, model_family +from senselab.audio.workflows.audio_analysis.adaptive.regions import region_buckets +from senselab.audio.workflows.audio_analysis.aggregate import _normalize_transcript_for_wer +from senselab.audio.workflows.audio_analysis.grid import BucketGrid +from senselab.audio.workflows.audio_analysis.harvesters import ( + _levenshtein, + asr_alignment_score_in_window, + asr_text_in_window, + resolve_asr_result, + whisper_bucket_avg_logprob, +) + +# ── shared artifact/cache access ───────────────────────────────────────── + + +def load_outcomes_dir(run_dir: Path, stream: str, task_dir: str) -> dict[str, dict[str, Any]]: + """Load ``///*.json`` keyed by provenance.model_id. + + Each payload records its ``_file_stem`` — alignment files are keyed by the + *aligner* model id in provenance but written under the parent ASR model's + safe filename, so cross-task joins go through the stem. + """ + out: dict[str, dict[str, Any]] = {} + d = run_dir / stream / task_dir + if not d.is_dir(): + return out + for f in sorted(d.glob("*.json")): + try: + payload = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError): + continue + payload["_file_stem"] = f.stem + model_id = ((payload.get("provenance") or {}).get("model_id")) or f.stem + out[str(model_id)] = payload + return out + + +def load_alignments_matched( + run_dir: Path, stream: str, asr_by_model: dict[str, dict[str, Any]] +) -> dict[str, dict[str, Any]]: + """Alignment payloads re-keyed by their parent **ASR** model id (stem join).""" + by_stem: dict[str, dict[str, Any]] = {} + d = run_dir / stream / "alignment" + if d.is_dir(): + for f in sorted(d.glob("*.json")): + try: + by_stem[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError): + continue + out: dict[str, dict[str, Any]] = {} + for model, block in asr_by_model.items(): + stem = block.get("_file_stem") + if stem and stem in by_stem: + out[model] = by_stem[stem] + return out + + +def build_cache_index(cache_dir: Path | None) -> dict[tuple[str, str, str], list[dict[str, Any]]]: + """Index cache entries by (audio_signature, task, model_id); values sorted by filename.""" + index: dict[tuple[str, str, str], list[dict[str, Any]]] = {} + if cache_dir is None or not Path(cache_dir).is_dir(): + return index + for f in sorted(Path(cache_dir).glob("*.json")): + try: + if f.stat().st_size > 30_000_000: # features blobs — never needed here + continue + payload = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError): + continue + prov = payload.get("provenance") or {} + sig, task = str(prov.get("audio_signature") or ""), str(prov.get("task") or "") + model = str(prov.get("model_id") or "") + if sig and task: + payload["_cache_file"] = f.name + index.setdefault((sig, task, model), []).append(payload) + return index + + +def _pick_ok(entries: list[dict[str, Any]]) -> dict[str, Any] | None: + for e in entries: # already filename-sorted → deterministic + if e.get("status") == "ok" and e.get("result") is not None: + return e + return None + + +def _has_g2p() -> bool: + try: + return importlib.util.find_spec("g2p_en") is not None + except (ImportError, ValueError): + return False + + +def _spec_missing(module: str) -> bool: + try: + return importlib.util.find_spec(module) is None + except (ImportError, ValueError, ModuleNotFoundError): + return True + + +# ── S1: stream election (C5, FR-015) ──────────────────────────────────── + + +def _rows_in_span(rows: list[dict[str, Any]], start: float, end: float) -> list[dict[str, Any]]: + return [r for r in rows if r["end"] > start and r["start"] < end] + + +def _mean(vals: list[float | None]) -> float | None: + clean = [v for v in vals if v is not None and v == v] + return sum(clean) / len(clean) if clean else None + + +def _election_scores(region: dict[str, Any], ctx: dict[str, Any]) -> dict[str, dict[str, float]]: + scores: dict[str, dict[str, float]] = {} + w = ctx["policy"]["election"]["weights"] + for stream in ctx["passes"]: + pres = _rows_in_span(ctx["state"].axis_rows(stream, "presence"), region["core_start"], region["core_end"]) + utt = _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), region["core_start"], region["core_end"]) + p_conf = _mean([r.get("p_voice") for r in pres]) or 0.0 + degr = _mean( + [ + max( + float((r.get("meta") or {}).get("quality_snr") or 0.0), + float((r.get("meta") or {}).get("quality_clip") or 0.0), + float((r.get("meta") or {}).get("quality_reverb") or 0.0), + ) + for r in pres + ] + ) + quality = 1.0 - (degr if degr is not None else 0.0) + agree = 1.0 - (_mean([r.get("aggregated_uncertainty") for r in utt]) or 0.0) + total = w["presence_conf"] * p_conf + w["quality"] * quality + w["utterance_agreement"] * agree + scores[stream] = { + "presence_conf": round(p_conf, 6), + "quality": round(quality, 6), + "utterance_agreement": round(agree, 6), + "total": round(total, 9), + } + return scores + + +def _s1_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or len(ctx["passes"]) < 2: + return False, {} + if region["region_id"] in ctx["elections"]: + return False, {} + return True, {"streams": list(ctx["passes"])} + + +def _s1_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + region = cand["region"] + scores = _election_scores(region, ctx) + elected = max(scores, key=lambda s: (scores[s]["total"], s == "raw_16k")) + guard_fired = False + if elected != "raw_16k": + # Enhancement-artifact guard (degraded): reject the enhanced stream when + # it claims speech text where the raw stream has none *and* raw presence + # is confidently silent — SepFormer can synthesize speech-like energy. + raw_text = _region_text(ctx, "raw_16k", region) + enh_text = _region_text(ctx, elected, region) + raw_pres = _mean( + [ + r.get("p_voice") + for r in _rows_in_span( + ctx["state"].axis_rows("raw_16k", "presence"), region["core_start"], region["core_end"] + ) + ] + ) + if enh_text and not raw_text and (raw_pres is not None and raw_pres < 0.2): + guard_fired = True + elected = "raw_16k" + election = { + "region_id": region["region_id"], + "scores": scores, + "elected": elected, + "guard_fired": guard_fired, + } + ctx["elections"][region["region_id"]] = election + region["elected_stream"] = elected + return {"election": election, "touched": {}} + + +def _region_text(ctx: dict[str, Any], stream: str, region: dict[str, Any]) -> str: + texts = [] + for row in _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), region["core_start"], region["core_end"]): + bk = bucket_key(row["start"], row["end"]) + for source, payload in ctx["store"].active_votes(stream, "utterance", bk).items(): + if not source.startswith("__") and payload.get("text"): + texts.append(str(payload["text"])) + return " ".join(texts).strip() + + +# ── P3 / C9: adjudication over existing evidence (C10 / C9) ───────────── + + +def _adjudication_candidates(ctx: dict[str, Any], stream: str) -> list[dict[str, Any]]: + adj = ctx["policy"]["adjudication"] + out = [] + for row in ctx["state"].axis_rows(stream, "presence"): + p_voice = row.get("p_voice") + if p_voice is None or p_voice >= adj["p_voice_hallucination"]: + continue + bk = bucket_key(row["start"], row["end"]) + votes = ctx["store"].active_votes(stream, "presence", bk) + for source, payload in votes.items(): + if source.startswith(("__", "acoustic_", "frame_", "embedding_", "ast", "yamnet")): + continue + if source not in ctx["asr_model_ids"].get(stream, set()): + continue + if not payload.get("speaks"): + continue + meta = row.get("meta") or {} + nc = payload.get("native_confidence") + indicators = { + "low_native_confidence": nc is not None and float(nc) < adj["low_native_confidence"], + "low_src_speech": (meta.get("src_speech") is not None) + and float(meta.get("src_speech") or 1.0) < adj["low_src_speech"], + "very_low_p_voice": float(p_voice) < adj["p_voice_hallucination"] / 2.0, + } + if sum(indicators.values()) >= adj["min_indicators"]: + out.append({"bucket": bk, "source": source, "indicators": indicators, "p_voice": p_voice}) + return out + + +def _p3_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + cands = {s: _adjudication_candidates(ctx, s) for s in ctx["passes"]} + n = sum(len(v) for v in cands.values()) + return n > 0, {"n_candidates": n} + + +def _p3_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + purged = [] + for stream in ctx["passes"]: + for c in _adjudication_candidates(ctx, stream): + n = ctx["store"].purge_source_in_bucket( + stream, c["bucket"], c["source"], reason="hallucination_adjudicated", round_idx=ctx["round_idx"] + ) + # Purge also hits utterance buckets overlapping this presence bucket. + for urow in _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), c["bucket"][0], c["bucket"][1]): + ubk = bucket_key(urow["start"], urow["end"]) + n += ctx["store"].purge_source_in_bucket( + stream, ubk, c["source"], reason="hallucination_adjudicated", round_idx=ctx["round_idx"] + ) + touched.setdefault((stream, "utterance"), set()).add(ubk) + touched.setdefault((stream, "presence"), set()).add(c["bucket"]) + purged.append({**c, "stream": stream, "votes_purged": n}) + return {"purged": purged, "touched": touched} + + +def _c9_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + n = sum(len(_missed_speech_candidates(ctx, s)) for s in ctx["passes"]) + return n > 0, {"n_candidates": n} + + +def _missed_speech_candidates(ctx: dict[str, Any], stream: str) -> list[dict[str, Any]]: + adj = ctx["policy"]["adjudication"] + out = [] + for row in ctx["state"].axis_rows(stream, "presence"): + p_voice = row.get("p_voice") + if p_voice is None or not (adj["p_voice_hallucination"] <= p_voice < adj["p_voice_missed"]): + continue + bk = bucket_key(row["start"], row["end"]) + if "adjudicator/missed_speech" in ctx["store"].active_votes(stream, "presence", bk): + continue + families = set() + for urow in _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), bk[0], bk[1]): + ubk = bucket_key(urow["start"], urow["end"]) + for source, payload in ctx["store"].active_votes(stream, "utterance", ubk).items(): + if not source.startswith("__") and payload.get("text"): + families.add(model_family(source, ctx["policy"])) + if len(families) >= 2: + out.append({"bucket": bk, "families": sorted(families), "p_voice": p_voice}) + return out + + +def _c9_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + added = [] + weight = float(ctx["policy"]["adjudication"]["missed_speech_weight"]) + for stream in ctx["passes"]: + for c in _missed_speech_candidates(ctx, stream): + ctx["store"].add_vote( + Vote( + axis="presence", + bucket=c["bucket"], + source="adjudicator/missed_speech", + stream=stream, + scope="file", + round=ctx["round_idx"], + payload={"speaks": True, "native_confidence": None, "weight": weight}, + provenance={"families_agreeing": c["families"], "rule": "C9_missed_speech"}, + ) + ) + touched.setdefault((stream, "presence"), set()).add(c["bucket"]) + added.append({**c, "stream": stream}) + return {"added": added, "touched": touched} + + +# ── U2: reserve escalation via cache replay ────────────────────────────── + + +def _reserves_in_cache(ctx: dict[str, Any], stream: str) -> list[str]: + sig = ctx["pass_sigs"].get(stream, "") + found = [] + for model in ctx["policy"].get("reserve_asr_models") or []: + if _pick_ok(ctx["cache_index"].get((sig, "asr", model), [])): + found.append(model) + return found + + +def _u2_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or region["axis"] != "utterance": + return False, {} + stream = region.get("elected_stream") or region.get("stream") or "raw_16k" + reserves = _reserves_in_cache(ctx, stream) + if not reserves: + return False, {"reason": "no_cached_reserves"} + rows = _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), region["core_start"], region["core_end"]) + epi = _mean([r.get("epistemic") for r in rows]) + if epi is None or epi < ctx["policy"]["thresholds"]["theta_low"]: + return False, {"epistemic": epi} + return True, {"reserves": reserves, "epistemic": round(float(epi), 6), "stream": stream} + + +def _u2_gain(region: dict[str, Any], ctx: dict[str, Any], trigger: dict[str, Any]) -> float: + return float(region["uncertainty_mass"]) * float(trigger.get("epistemic") or 0.0) * 10.0 + + +def _u2_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + region, trigger = cand["region"], cand["trigger"] + stream = trigger["stream"] + sig = ctx["pass_sigs"][stream] + run_dir: Path = ctx["run_dir"] + + asr_by_model = dict(load_outcomes_dir(run_dir, stream, "asr")) + align_by_model = load_alignments_matched(run_dir, stream, asr_by_model) + used_reserves: list[dict[str, Any]] = [] + for model in trigger["reserves"]: + entry = _pick_ok(ctx["cache_index"].get((sig, "asr", model), [])) + if entry is None: + continue + asr_by_model[model] = entry + # Text-only reserves need their cached alignment for word timestamps. + parent_key = entry.get("cache_key") or (entry.get("provenance") or {}).get("cache_key") + for (esig, task, amodel), align_entries in ctx["cache_index"].items(): + if esig != sig or task != "alignment": + continue + match = next( + ( + a + for a in align_entries + if a.get("status") == "ok" + and (a.get("provenance") or {}).get("parent_asr_cache_key") + and (a.get("provenance") or {}).get("parent_asr_cache_key") == parent_key + ), + None, + ) + if match is not None: + align_by_model[model] = match + break + used_reserves.append({"model": model, "cache_file": entry.get("_cache_file")}) + + pass_summary_ext = {"duration_s": ctx["duration_s"], "asr": {"by_model": asr_by_model}} + grid = BucketGrid(win_length=ctx["utterance_grid"][0], hop_length=ctx["utterance_grid"][1]) + + pair_kind = "phoneme" + if _has_g2p(): + from senselab.audio.workflows.audio_analysis.utterance import harvest_utterance_votes + + harvested = harvest_utterance_votes( + pass_summary=pass_summary_ext, grid=grid, ppg_block={}, alignment_by_model=align_by_model + ) + else: + pair_kind = "word" + harvested = _harvest_word_level(pass_summary_ext, grid, align_by_model, ctx["duration_s"]) + + rows = ctx["state"].axis_rows(stream, "utterance") + covered = region_buckets(region, rows) + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + n_votes = 0 + for entry in harvested: + bk = bucket_key(entry["start"], entry["end"]) + if bk not in covered: + continue # merge-back midpoint rule (D2) + for source, payload in entry["votes"].items(): + if isinstance(payload, dict) and source == "__pairwise_phoneme_distances__": + payload = {**payload, "pair_distance_kind": pair_kind} + ctx["store"].add_vote( + Vote( + axis="utterance", + bucket=bk, + source=source, + stream=stream, + scope=f"region:{region['region_id']}", + round=ctx["round_idx"], + payload=payload if isinstance(payload, dict) else {"value": payload}, + provenance={"rule": "U2_reserve_escalation", "reserves": used_reserves}, + ) + ) + n_votes += 1 + touched.setdefault((stream, "utterance"), set()).add(bk) + fam = family_weights(sorted([m for m in asr_by_model]), ctx["policy"]) + return { + "reserves_used": used_reserves, + "pair_distance_kind": pair_kind, + "votes_added": n_votes, + "family_weights": fam, + "touched": touched, + } + + +def _harvest_word_level( + pass_summary: dict[str, Any], + grid: BucketGrid, + alignment_by_model: dict[str, Any], + duration_s: float, +) -> list[dict[str, Any]]: + """g2p-free fallback: pairwise WORD-Levenshtein rate (the original FR-002 WER form). + + Same vote schema as ``harvest_utterance_votes`` so ``aggregate_utterance`` + consumes it unchanged; ``pair_distance_kind: "word"`` is recorded on the + pair block by the caller. + """ + from itertools import combinations + + asr_blocks = (pass_summary.get("asr") or {}).get("by_model") or {} + resolved = { + m: resolve_asr_result(b, alignment_by_model.get(m)) + for m, b in asr_blocks.items() + if isinstance(b, dict) and b.get("status") == "ok" + } + out = [] + for start, end, _idx in grid.iter_buckets(duration_s): + votes: dict[str, Any] = {} + seqs: dict[str, list[str]] = {} + confs: dict[str, float] = {} + for m, res in resolved.items(): + text = asr_text_in_window(res, start, end, fully_contained=True) + alp = whisper_bucket_avg_logprob(res, start, end) + ctc = asr_alignment_score_in_window(res, start, end) + votes[m] = {"text": text, "phoneme_sequence": [], "avg_logprob": alp, "alignment_ctc_score": ctc} + tokens = _normalize_transcript_for_wer(text or "").split() + if tokens: + seqs[m] = tokens + if alp is not None: + try: + confs[m] = max(0.0, min(1.0, math.exp(float(alp)))) + except (ValueError, OverflowError): + pass + pairs = {} + for a, b in combinations(sorted(seqs), 2): + denom = max(len(seqs[a]), len(seqs[b])) + if denom: + pairs[f"{a}|{b}"] = min(1.0, _levenshtein(seqs[a], seqs[b]) / denom) + votes["__pairwise_phoneme_distances__"] = { + "pairs": pairs, + "n_sources": len(seqs), + "sources": sorted(seqs), + "per_source_confidence": confs, + } + out.append({"start": start, "end": end, "votes": votes}) + return out + + +# ── U1: live region re-ASR ─────────────────────────────────────────────── + + +def _u1_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or region["axis"] != "utterance": + return False, {} + models = [m for m in (ctx["policy"].get("u1_asr_models") or []) if m not in ctx.get("live_asr_done", set())] + if not models: + return False, {"reason": "no_u1_models"} + stream = region.get("elected_stream") or region.get("stream") or "raw_16k" + rows = _rows_in_span(ctx["state"].axis_rows(stream, "utterance"), region["core_start"], region["core_end"]) + epi = _mean([r.get("epistemic") for r in rows]) + if epi is None or epi < ctx["policy"]["thresholds"]["theta_low"]: + return False, {"epistemic": epi} + return True, { + "models": models, + "stream": stream, + "epistemic": round(float(epi), 6), + "crop": [region["crop_start"], region["crop_end"]], + } + + +def _u1_guard(region: dict[str, Any], ctx: dict[str, Any]) -> str | None: + if _spec_missing("transformers") and not _senselab_asr_available(): + return "asr_backend_unavailable" + if not ctx.get("input_audio"): + return "input_audio_missing" + return None + + +def _senselab_asr_available() -> bool: + from senselab.audio.workflows.audio_analysis.adaptive.backends import senselab_transcribe_available + + return senselab_transcribe_available() + + +def _u1_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + from senselab.audio.workflows.audio_analysis.adaptive.audio_io import crop as crop_wav + from senselab.audio.workflows.audio_analysis.adaptive.audio_io import get_stream_wav + from senselab.audio.workflows.audio_analysis.adaptive.backends import transcribe_crop + + region, trigger = cand["region"], cand["trigger"] + stream = trigger["stream"] + wav, reason = get_stream_wav(ctx, stream) + stream_fallback = None + if wav is None and stream != "raw_16k": + stream_fallback = reason + stream = "raw_16k" + wav, reason = get_stream_wav(ctx, stream) + if wav is None: + raise RuntimeError(f"audio_unavailable: {reason}") + + crop_start, crop_end = float(region["crop_start"]), float(region["crop_end"]) + segment = crop_wav(wav, crop_start, crop_end) + language = ctx["policy"].get("language") + ran: list[dict[str, Any]] = [] + ext_blocks: dict[str, dict[str, Any]] = {} + u1_backend = str(ctx["policy"].get("u1_backend", "auto")) + for model in trigger["models"]: + meta: dict[str, Any] = {} + words, err = transcribe_crop( + segment, model_id=model, offset_s=crop_start, language=language, meta=meta, backend=u1_backend + ) + if words is None: + ran.append({"model": model, "status": "failed", "error": err}) + continue + ran.append({"model": model, "status": "ok", "n_words": len(words), **meta}) + ctx.setdefault("live_asr_done", set()).add(model) + ctx.setdefault("live_asr_words", {}).setdefault(stream, {})[model] = words + ext_blocks[model] = { + "status": "ok", + "result": [ + { + "text": " ".join(w["text"] for w in words), + "start": crop_start, + "end": crop_end, + "chunks": [{**w, "chunks": None} for w in words], + } + ], + } + if not ext_blocks: + raise RuntimeError(f"all_u1_models_failed: {ran}") + + # Merge with the run's own ASR and re-harvest the covered buckets — same + # path as U2 so votes/pairs stay schema-identical. + asr_by_model = dict(load_outcomes_dir(ctx["run_dir"], stream, "asr")) + align_by_model = load_alignments_matched(ctx["run_dir"], stream, asr_by_model) + asr_by_model.update(ext_blocks) + pass_summary_ext = {"duration_s": ctx["duration_s"], "asr": {"by_model": asr_by_model}} + grid = BucketGrid(win_length=ctx["utterance_grid"][0], hop_length=ctx["utterance_grid"][1]) + pair_kind = "phoneme" + if _has_g2p(): + from senselab.audio.workflows.audio_analysis.utterance import harvest_utterance_votes + + harvested = harvest_utterance_votes( + pass_summary=pass_summary_ext, grid=grid, ppg_block={}, alignment_by_model=align_by_model + ) + else: + pair_kind = "word" + harvested = _harvest_word_level(pass_summary_ext, grid, align_by_model, ctx["duration_s"]) + rows = ctx["state"].axis_rows(stream, "utterance") + covered = region_buckets(region, rows) + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + n_votes = 0 + for entry in harvested: + bk = bucket_key(entry["start"], entry["end"]) + if bk not in covered: + continue + for source, payload in entry["votes"].items(): + if isinstance(payload, dict) and source == "__pairwise_phoneme_distances__": + payload = {**payload, "pair_distance_kind": pair_kind} + ctx["store"].add_vote( + Vote( + axis="utterance", + bucket=bk, + source=source, + stream=stream, + scope=f"region:{region['region_id']}", + round=ctx["round_idx"], + payload=payload if isinstance(payload, dict) else {"value": payload}, + provenance={"rule": "U1_region_reasr", "models": ran, "stream_fallback": stream_fallback}, + ) + ) + n_votes += 1 + touched.setdefault((stream, "utterance"), set()).add(bk) + return { + "models": ran, + "stream": stream, + "stream_fallback": stream_fallback, + "pair_distance_kind": pair_kind, + "votes_added": n_votes, + "touched": touched, + } + + +# ── I1 + I2: identity repair from embeddings (stored artifacts or live) ─ + + +def _get_identity_repair(ctx: dict[str, Any], stream: str) -> dict[str, Any] | None: + """Compute (once per stream) the change-point + recluster repair.""" + cache = ctx.setdefault("_identity_repair", {}) + if stream in cache: + return cache[stream] + from senselab.audio.workflows.audio_analysis.adaptive.fusion import make_p_voice_lookup + from senselab.audio.workflows.audio_analysis.adaptive.identity_repair import repair_identity + + window_embeddings: dict[str, list[dict[str, Any]]] = {} + emb_dir = ctx["run_dir"] / stream / "embeddings" + if emb_dir.is_dir(): + for f in sorted(emb_dir.glob("*.json")): + try: + payload = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError): + continue + windows = payload.get("windows") or [] + if len(windows) >= 4: + window_embeddings[f.stem] = windows + diar_boundaries: list[float] = [] + for block in load_outcomes_dir(ctx["run_dir"], stream, "diarization").values(): + res = block.get("result") + if isinstance(res, list) and res: + inner = res[0] if isinstance(res[0], list) else res + for seg in inner: + if isinstance(seg, dict): + for key in ("start", "end"): + v = seg.get(key) + if v is not None and 0.05 < float(v) < ctx["duration_s"] - 0.05: + diar_boundaries.append(float(v)) + repaired = None + if window_embeddings: + repaired = repair_identity( + window_embeddings=window_embeddings, + diar_boundaries=diar_boundaries, + p_voice_at=make_p_voice_lookup(ctx["state"], stream), + duration_s=ctx["duration_s"], + policy=ctx["policy"], + ) + cache[stream] = repaired + return repaired + + +def _i1_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or region["axis"] != "identity": + return False, {} + stream = region.get("stream") or "raw_16k" + if (stream, region["core_start"], region["core_end"]) in ctx.get("_i1_done", set()): + return False, {} + return True, {"crop": [region["crop_start"], region["crop_end"]], "stream": stream} + + +def _i1_guard(region: dict[str, Any], ctx: dict[str, Any]) -> str | None: + emb_dir = ctx["run_dir"] / (region.get("stream") or "raw_16k") / "embeddings" + if not emb_dir.is_dir() or not any(emb_dir.glob("*.json")): + if _spec_missing("torch") or _spec_missing("speechbrain"): + return "embedding_backend_unavailable (no stored embeddings, no live backend)" + return None + + +def _i1_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + region = cand["region"] + stream = cand["trigger"]["stream"] + repaired = _get_identity_repair(ctx, stream) + if repaired is None: + raise RuntimeError("identity_repair_no_embeddings") + ctx.setdefault("_i1_done", set()).add((stream, region["core_start"], region["core_end"])) + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + n_votes = 0 + in_region = [c for c in repaired["change_points"] if region["crop_start"] <= c["time"] <= region["crop_end"]] + for row in _rows_in_span(ctx["state"].axis_rows(stream, "identity"), region["core_start"], region["core_end"]): + bk = bucket_key(row["start"], row["end"]) + cps_here = [c for c in in_region if row["start"] <= c["time"] < row["end"]] + if not cps_here: + continue + ctx["store"].add_vote( + Vote( + axis="identity", + bucket=bk, + source="embedding_changepoint/consensus", + stream=stream, + scope="file", + round=ctx["round_idx"], + payload={ + "change_point_times": [c["time"] for c in cps_here], + "change_point_confidence": max(c["confidence"] for c in cps_here), + }, + provenance={"rule": "I1_boundary_refinement", "models": repaired["models_used"]}, + ) + ) + touched.setdefault((stream, "identity"), set()).add(bk) + n_votes += 1 + return { + "change_points_in_region": in_region, + "votes_added": n_votes, + "models": repaired["models_used"], + "touched": touched, + } + + +def _i2_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or region["axis"] != "identity": + return False, {} + stream = region.get("stream") or "raw_16k" + if ctx.get("refined_identity", {}).get(stream): + return False, {} # once per stream per run + return True, {"stream": stream} + + +def _i2_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + from senselab.audio.workflows.audio_analysis.adaptive.identity_repair import ( + cluster_at, + cross_source_disagreement, + ) + + stream = cand["trigger"]["stream"] + repaired = _get_identity_repair(ctx, stream) + if repaired is None: + raise RuntimeError("identity_repair_no_embeddings") + ctx.setdefault("refined_identity", {})[stream] = repaired + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + n_votes = 0 + for row in ctx["state"].axis_rows(stream, "identity"): + bk = bucket_key(row["start"], row["end"]) + mid = (row["start"] + row["end"]) / 2.0 + cid = cluster_at(repaired, mid) + prev_mid = mid - (row["end"] - row["start"]) + changed = cluster_at(repaired, prev_mid) != cid if prev_mid >= 0 else False + ctx["store"].add_vote( + Vote( + axis="identity", + bucket=bk, + source="embedding_recluster/consensus", + stream=stream, + scope="file", + round=ctx["round_idx"], + payload={ + "speaker_label": cid or "SIL", + "cluster_id": cid or "SIL", + "speaker_changed_from_prev": changed, + }, + provenance={"rule": "I2_recluster", "n_clusters": repaired["n_clusters"]}, + ) + ) + # Recompute cross-source label disagreement including the new voter + # (overwrites the file-scope vote deterministically — same vote id). + ids = [] + for source, payload in ctx["store"].active_votes(stream, "identity", bk).items(): + if source.startswith("__") or "::" in source: + continue + c = payload.get("cluster_id") + if c and c not in ("SIL", ""): + ids.append(str(c)) + value = cross_source_disagreement(ids) + if value is not None: + ctx["store"].add_vote( + Vote( + axis="identity", + bucket=bk, + source="__cross_diar_label_disagreement__", + stream=stream, + scope="file", + round=ctx["round_idx"], + payload={"value": value, "n_sources": len(ids), "recomputed_by": "I2_recluster"}, + provenance={"rule": "I2_recluster"}, + ) + ) + touched.setdefault((stream, "identity"), set()).add(bk) + n_votes += 2 + return { + "n_clusters": repaired["n_clusters"], + "n_segments": len(repaired["segments"]), + "change_points": len(repaired["change_points"]), + "models": repaired["models_used"], + "votes_added": n_votes, + "touched": touched, + } + + +# ── I4: overlap posteriors (gated live backend) ───────────────────────── + + +def _i4_trigger(region: dict[str, Any], ctx: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + if region is None or region["axis"] != "identity": + return False, {} + co = [ + r + for r in ctx.get("all_regions", []) + if r["axis"] == "utterance" + and r.get("stream") == region.get("stream") + and min(r["core_end"], region["core_end"]) - max(r["core_start"], region["core_start"]) > 0 + ] + return bool(co), { + "co_located_utterance_regions": [r["region_id"] for r in co], + "stream": region.get("stream") or "raw_16k", + } + + +def _i4_guard(region: dict[str, Any], ctx: dict[str, Any]) -> str | None: + import os + + if _spec_missing("pyannote"): + return "posteriors_unavailable (pyannote.audio not installed)" + if not (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")): + return "posteriors_unavailable (HF token required for pyannote/segmentation-3.0)" + if not ctx.get("input_audio"): + return "input_audio_missing" + return None + + +def _i4_execute(cand: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]: + from senselab.audio.workflows.audio_analysis.adaptive.audio_io import get_stream_wav + from senselab.audio.workflows.audio_analysis.adaptive.backends import overlap_posteriors + + region = cand["region"] + stream = cand["trigger"]["stream"] + # Overlap is a property of the scene, not the processing stream: fall back + # to the raw waveform when the region's stream can't be materialized, and + # apply the posterior to every stream's rows over the span. + wav, reason = get_stream_wav(ctx, stream) + source_stream, stream_fallback = stream, None + if wav is None and stream != "raw_16k": + stream_fallback = reason + source_stream = "raw_16k" + wav, reason = get_stream_wav(ctx, source_stream) + if wav is None: + raise RuntimeError(f"audio_unavailable: {reason}") + post, err = overlap_posteriors(wav, span=(region["crop_start"], region["crop_end"])) + if post is None: + raise RuntimeError(err or "posteriors_failed") + touched: dict[tuple[str, str], set[tuple[float, float]]] = {} + hop, overlap = float(post["frame_hop"]), post["overlap"] + + def _mean_overlap(s: float, e: float) -> float | None: + lo = max(0, int((s - region["crop_start"]) / hop)) + hi = min(len(overlap), int((e - region["crop_start"]) / hop) + 1) + vals = overlap[lo:hi] + return float(sum(vals) / len(vals)) if vals else None + + mean_over_core = _mean_overlap(region["core_start"], region["core_end"]) + for apply_stream in ctx["passes"]: + for axis in ("identity", "utterance"): + for row in _rows_in_span( + ctx["state"].axis_rows(apply_stream, axis), region["core_start"], region["core_end"] + ): + ov = _mean_overlap(row["start"], row["end"]) + if ov is None: + continue + row.setdefault("meta", {})["overlap_posterior"] = round(ov, 4) + row["overlap_posterior"] = round(ov, 4) + touched.setdefault((apply_stream, axis), set()).add(bucket_key(row["start"], row["end"])) + return { + "source_stream": source_stream, + "stream_fallback": stream_fallback, + "frames": len(overlap), + "mean_overlap_in_core": round(mean_over_core, 4) if mean_over_core is not None else None, + "n_classes": post.get("n_classes"), + "touched": touched, + } + + +def _mass_gain(region: dict[str, Any], ctx: dict[str, Any], trigger: dict[str, Any]) -> float: + return float(region["uncertainty_mass"]) if region else 0.0 + + +def _n_candidates_gain(region: dict[str, Any], ctx: dict[str, Any], trigger: dict[str, Any]) -> float: + return float(trigger.get("n_candidates") or 0) + + +RULES: list[dict[str, Any]] = [ + { + "id": "S1_stream_election", + "axes": ["presence", "identity", "utterance"], + "cost": "light", + "trigger": _s1_trigger, + "guard": None, + "gain": _mass_gain, + "execute": _s1_execute, + }, + { + "id": "P3_hallucination_adjudication", + "axes": [], # stream-global, runs at most once per round + "meta_axis": "presence", + "cost": "light", + "trigger": _p3_trigger, + "guard": None, + "gain": _n_candidates_gain, + "execute": _p3_execute, + }, + { + "id": "C9_missed_speech", + "axes": [], + "meta_axis": "presence", + "cost": "light", + "trigger": _c9_trigger, + "guard": None, + "gain": _n_candidates_gain, + "execute": _c9_execute, + }, + { + "id": "U2_reserve_escalation", + "axes": ["utterance"], + "cost": "medium", + "trigger": _u2_trigger, + "guard": None, + "gain": _u2_gain, + "execute": _u2_execute, + }, + { + "id": "U1_region_reasr", + "axes": ["utterance"], + "cost": "medium", + "trigger": _u1_trigger, + "guard": _u1_guard, + "gain": _mass_gain, + "execute": _u1_execute, + }, + { + "id": "I1_boundary_refinement", + "axes": ["identity"], + "cost": "light", + "trigger": _i1_trigger, + "guard": _i1_guard, + "gain": _mass_gain, + "execute": _i1_execute, + }, + { + "id": "I2_recluster", + "axes": ["identity"], + "cost": "light", + "trigger": _i2_trigger, + "guard": _i1_guard, # same requirement: stored embeddings or live backend + "gain": _mass_gain, + "execute": _i2_execute, + }, + { + "id": "I4_overlap_detection", + "axes": ["identity"], + "cost": "medium", + "trigger": _i4_trigger, + "guard": _i4_guard, + "gain": _mass_gain, + "execute": _i4_execute, + }, +] diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/loop.py b/src/senselab/audio/workflows/audio_analysis/adaptive/loop.py new file mode 100644 index 000000000..c3234404c --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/loop.py @@ -0,0 +1,470 @@ +"""Orchestrator: ingest → belief → rounds of policy-ranked interventions → fusion. + +Prototype entry point (``run_adaptive_loop``). Artifact-driven: round 1 is the +ingested analyze_audio run; rounds 2..K execute the intervention catalog with +budget + convergence semantics from the spec; the final round fuses. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + +from senselab.audio.workflows.audio_analysis.adaptive.belief import AXES, BeliefState, VoteStore +from senselab.audio.workflows.audio_analysis.adaptive.convergence import ( + apply_convergence_marks, + build_convergence_report, + round_summary, +) +from senselab.audio.workflows.audio_analysis.adaptive.fusion import ( + build_final_outputs, + collect_word_streams, + fuse_words, + make_p_voice_lookup, + make_speaker_lookup, +) +from senselab.audio.workflows.audio_analysis.adaptive.interventions import ( + RULES, + _pick_ok, + build_cache_index, + load_alignments_matched, + load_outcomes_dir, +) +from senselab.audio.workflows.audio_analysis.adaptive.policy import BudgetLedger, load_policy, plan_round +from senselab.audio.workflows.audio_analysis.adaptive.regions import propose_regions + + +def run_adaptive_loop( + run_dir: Path, + *, + cache_dir: Path | None = None, + policy_path: Path | None = None, + out_dir: Path | None = None, + max_rounds: int = 3, + aggregator: str | None = None, +) -> dict[str, Any]: + """Run the adaptive loop over a completed analyze_audio run directory.""" + run_dir = Path(run_dir) + out_dir = Path(out_dir) if out_dir else run_dir + policy = load_policy(policy_path) + + summary = json.loads((run_dir / "summary.json").read_text()) + passes = [pl for pl, ps in (summary.get("passes") or {}).items() if isinstance(ps, dict) and "duration_s" in ps] + if not passes: + raise ValueError(f"no completed passes in {run_dir}/summary.json") + duration_s = float(summary["passes"][passes[0]]["duration_s"]) + pass_sigs = {pl: str(summary["passes"][pl].get("audio_signature") or "") for pl in passes} + if aggregator is None: + aggregator = _aggregator_from_run(run_dir) or "min" + + # ── round 1: ingest + parity (harvest/aggregate split proof) ──────── + t0 = time.time() + store = VoteStore.from_run_dir(run_dir, passes) + parity = store.parity_check(passes, aggregator=aggregator) + state = BeliefState.from_store(store, passes, aggregator=aggregator) + utterance_grid = _grid_from_rows(state.axis_rows(passes[0], "utterance")) + theta_low = float(policy["thresholds"]["theta_low"]) + + rounds_dir = out_dir / "rounds" + _write_round_belief(rounds_dir / "1", state, passes) + (rounds_dir / "1" / "summary.json").write_text( + json.dumps( + { + "round": 1, + "ingested_from": str(run_dir), + "parity_check": parity, + "aggregator": aggregator, + "uncertainty_mass": { + f"{s}/{a}": round(state.uncertainty_mass(s, a, theta_low), 6) for s in passes for a in AXES + }, + }, + indent=2, + ) + ) + + input_audio = _resolve_input_audio(summary.get("input_audio"), run_dir) + ctx: dict[str, Any] = { + "store": store, + "state": state, + "policy": policy, + "run_dir": run_dir, + "cache_index": build_cache_index(cache_dir), + "passes": passes, + "pass_sigs": pass_sigs, + "duration_s": duration_s, + "utterance_grid": utterance_grid, + "elections": {}, + "input_audio": input_audio, + "asr_model_ids": {pl: set(load_outcomes_dir(run_dir, pl, "asr").keys()) for pl in passes}, + } + + ledger = BudgetLedger(policy) + iterations: list[dict[str, Any]] = [] + round_summaries: list[dict[str, Any]] = [] + touch_counts: dict[tuple[str, str, tuple[float, float]], int] = {} + run_state = "max_rounds" + + # ── rounds 2..K ────────────────────────────────────────────────────── + for round_idx in range(2, max_rounds + 1): + ctx["round_idx"] = round_idx + mass_before = {f"{s}/{a}": round(state.uncertainty_mass(s, a, theta_low), 6) for s in passes for a in AXES} + regions: list[dict[str, Any]] = [] + for stream in passes: + for axis in AXES: + regions.extend( + propose_regions( + state.axis_rows(stream, axis), + axis=axis, + stream=stream, + policy=policy, + round_idx=round_idx, + duration_s=duration_s, + ) + ) + ctx["all_regions"] = regions + admitted, not_admitted = plan_round( + rules=RULES, regions=regions, ctx=ctx, ledger=ledger, policy=policy, round_idx=round_idx + ) + + fired: list[dict[str, Any]] = [] + for cand in admitted: + rule = next(r for r in RULES if r["id"] == cand["rule"]) + entry = _iteration_entry(cand, round_idx) + try: + before_vals = _bucket_values(state) + result = rule["execute"](cand, ctx) + touched: dict[tuple[str, str], set] = result.pop("touched", {}) + delta = {} + for (stream, axis), buckets in touched.items(): + state.update_buckets(store, stream, axis, buckets, round_idx) + for bk in buckets: + key = (stream, axis, bk) + touch_counts[key] = touch_counts.get(key, 0) + 1 + # Before/after means over the SAME (touched) bucket set. + befores = [v for bk in buckets if (v := before_vals.get((stream, axis, bk))) is not None] + after = _mean_over(state, stream, axis, buckets) + if befores and after is not None: + before = sum(befores) / len(befores) + delta[f"{stream}/{axis}"] = { + "mean_before": round(before, 6), + "mean_after": round(after, 6), + "delta": round(after - before, 6), + "n_buckets": len(buckets), + } + entry.update({"status": "fired", "result": _json_safe_result(result), "delta": delta}) + cand["exec_status"] = "ok" + except Exception as exc: # noqa: BLE001 — failure envelope (D11) + entry.update({"status": "failed", "error": repr(exc)}) + cand["exec_status"] = "failed" + iterations.append(entry) + fired.append(cand) + for cand in not_admitted: + iterations.append(_iteration_entry(cand, round_idx)) + + budget_left = ledger.can_admit("medium") or ledger.can_admit("heavy") + apply_convergence_marks(state, passes=passes, policy=policy, touch_counts=touch_counts, budget_left=budget_left) + rs = round_summary( + round_idx=round_idx, + state=state, + passes=passes, + policy=policy, + fired=fired, + not_admitted=not_admitted, + mass_before=mass_before, + ledger=ledger, + ) + round_summaries.append(rs) + rd = rounds_dir / str(round_idx) + _write_round_belief(rd, state, passes) + (rd / "regions.json").write_text(json.dumps(regions, indent=2, default=str)) + (rd / "summary.json").write_text(json.dumps(rs, indent=2, default=str)) + _write_round_votes(rd, store, round_idx) + + if not fired: + run_state = "converged" if not not_admitted else "no_runnable_interventions" + break + else: + run_state = "max_rounds" + + # ── fusion round ───────────────────────────────────────────────────── + # The consensus transcript comes from the stream whose FINAL utterance + # evidence is most self-consistent (lowest residual uncertainty mass); + # region elections break ties. Enhancement can degrade ASR even when it + # improves presence/quality signals, so transcript fusion must not + # inherit the presence/quality-weighted election blindly. + elected_streams = [e["elected"] for e in ctx["elections"].values()] + fusion_stream = min( + passes, + key=lambda s: ( + round(state.uncertainty_mass(s, "utterance", theta_low), 9), + -elected_streams.count(s), + s, + ), + ) + asr_by_model = load_outcomes_dir(run_dir, fusion_stream, "asr") + align_by_model = load_alignments_matched(run_dir, fusion_stream, asr_by_model) + # Reserve models pulled in by U2 join the fusion ensemble (cache replay). + for entry in iterations: + if entry["rule"] == "U2_reserve_escalation" and entry["status"] == "fired": + for res in (entry.get("result") or {}).get("reserves_used", []): + model = res["model"] + cached = _pick_ok(ctx["cache_index"].get((pass_sigs[fusion_stream], "asr", model), [])) + if cached is not None: + asr_by_model.setdefault(model, cached) + for (sig, task, _m), aligns in ctx["cache_index"].items(): + if sig == pass_sigs[fusion_stream] and task == "alignment": + m2 = next( + ( + a + for a in aligns + if (a.get("provenance") or {}).get("parent_asr_cache_key") + == (cached.get("cache_key") or (cached.get("provenance") or {}).get("cache_key")) + ), + None, + ) + if m2 is not None: + align_by_model.setdefault(model, m2) + purged_spans = [ + (p["bucket"][0], p["bucket"][1], p["source"]) + for entry in iterations + if entry["rule"] == "P3_hallucination_adjudication" and entry["status"] == "fired" + for p in (entry.get("result") or {}).get("purged", []) + ] + word_streams = collect_word_streams(asr_by_model, align_by_model, purged_spans=purged_spans) + # Live U1 words (already in file time) join the ensemble on their stream. + for model, live_words in (ctx.get("live_asr_words", {}).get(fusion_stream) or {}).items(): + if live_words and model not in word_streams: + word_streams[model] = sorted(live_words, key=lambda w: (w["start"], w["end"])) + + # Speaker attribution: refined identity clusters (I2) win where available; + # the vote-majority lookup is the fallback. + refined = (ctx.get("refined_identity") or {}).get(fusion_stream) + base_speaker_at = make_speaker_lookup(store, state, fusion_stream) + if refined is not None: + from senselab.audio.workflows.audio_analysis.adaptive.identity_repair import cluster_at + + def speaker_at(t: float) -> str | None: + return cluster_at(refined, t) or base_speaker_at(t) + else: + speaker_at = base_speaker_at + from senselab.audio.workflows.audio_analysis.adaptive.fusion import load_calibrator + + calibrator = load_calibrator(policy.get("calibration_profile")) + words = fuse_words( + word_streams, + policy=policy, + speaker_at=speaker_at, + p_voice_at=make_p_voice_lookup(state, fusion_stream), + calibrator=calibrator, + ) + + # U3 (C8): consensus re-alignment for authoritative word timestamps — + # guarded live aligner; fallback = weighted member timestamps. + timestamps_meta: dict[str, Any] = {"timestamps_source": "member_vote"} + fus_cfg = policy.get("fusion") or {} + if words and str(fus_cfg.get("consensus_alignment", "auto")) == "auto": + from senselab.audio.workflows.audio_analysis.adaptive.audio_io import get_stream_wav + from senselab.audio.workflows.audio_analysis.adaptive.backends import consensus_align + + wav, wav_reason = get_stream_wav(ctx, fusion_stream) + if wav is None: + timestamps_meta["reason"] = wav_reason + else: + aligned, align_reason = consensus_align( + wav, words, timeout_s=float(fus_cfg.get("consensus_alignment_timeout_s", 600.0)) + ) + if aligned is not None: + for w, ts in zip(words, aligned): + w["start"], w["end"] = ts["start"], ts["end"] + words.sort(key=lambda w: (w["start"], w["end"])) + timestamps_meta = {"timestamps_source": "consensus_alignment_mms_fa"} + else: + timestamps_meta["reason"] = align_reason + last_round = round_summaries[-1]["round"] if round_summaries else 1 + transcript_doc = build_final_outputs( + out_dir=out_dir, + words=words, + store=store, + state=state, + stream=fusion_stream, + policy=policy, + generated_from_round=last_round, + refined_identity=refined, + calibrated=calibrator is not None, + timestamps_meta=timestamps_meta, + language=policy.get("language"), + ) + + # T032: LS final tracks + resolved-disagreements index (best-effort, additive). + try: + import json as _json + + from senselab.audio.workflows.audio_analysis.adaptive.ls_final import build_final_ls_bundle + + diarization_doc = _json.loads((out_dir / "final" / "diarization.json").read_text()) + build_final_ls_bundle( + out_dir=out_dir, + run_dir=run_dir, + transcript=transcript_doc, + diarization=diarization_doc, + presence_rows=state.axis_rows(fusion_stream, "presence"), + fusion_stream=fusion_stream, + iterations=iterations, + ) + except Exception as exc: # noqa: BLE001 — sidecar must never fail the run + print(f"warn: LS final bundle failed: {exc!r}") + provenance = { + "policy_hash": policy.get("policy_hash"), + "aggregator": aggregator, + "run_dir": str(run_dir), + "fusion_stream": fusion_stream, + "audio_backend": ctx.get("audio_backend") or None, # loader per stream (T048 — never silent) + "elapsed_s": round(time.time() - t0, 3), + } + report = build_convergence_report( + state=state, + passes=passes, + policy=policy, + rounds=round_summaries, + ledger=ledger, + iterations=iterations, + run_state=run_state, + provenance=provenance, + ) + final = out_dir / "final" + final.mkdir(parents=True, exist_ok=True) + (final / "convergence.json").write_text(json.dumps(report, indent=2, default=str)) + (final / "iterations.json").write_text( + json.dumps({"policy_hash": policy.get("policy_hash"), "entries": iterations}, indent=2, default=str) + ) + return { + "run_state": run_state, + "parity_check": parity, + "rounds": len(round_summaries) + 1, + "n_interventions_fired": sum(1 for e in iterations if e["status"] == "fired"), + "n_words_fused": len(words), + "fusion_stream": fusion_stream, + "word_streams": word_streams, + "out_dir": str(out_dir), + "report": report, + } + + +# ── helpers ────────────────────────────────────────────────────────────── + + +def _resolve_input_audio(recorded: str | None, run_dir: Path) -> str | None: + """Resolve the run's input audio path, re-rooting when the run came from another machine. + + Tries the recorded absolute path first, then the last one/two path components + relative to the repo root inferred from ``run_dir`` (…/artifacts/e2e_runs/ + → repo). Returns None when nothing exists — audio-dependent rules then guard. + """ + if not recorded: + return None + p = Path(recorded) + if p.exists(): + return str(p) + candidates = [] + for base in (run_dir.parent, run_dir.parent.parent, run_dir.parent.parent.parent): + candidates.extend([base / Path(*p.parts[-2:]), base / p.name]) + for c in candidates: + if c.exists(): + return str(c) + return None + + +def _aggregator_from_run(run_dir: Path) -> str | None: + dis = run_dir / "disagreements.json" + if dis.exists(): + try: + return (json.loads(dis.read_text()).get("config") or {}).get("aggregator") + except (OSError, json.JSONDecodeError): + return None + return None + + +def _grid_from_rows(rows: list[dict[str, Any]]) -> tuple[float, float]: + if not rows: + return (1.0, 0.5) + win = float(rows[0]["end"]) - float(rows[0]["start"]) + hop = (float(rows[1]["start"]) - float(rows[0]["start"])) if len(rows) > 1 else win + return (round(win, 6), round(hop, 6) or win) + + +def _bucket_values(state: BeliefState) -> dict[tuple[str, str, tuple[float, float]], float | None]: + """Per-bucket aggregated uncertainty snapshot (delta baselines use the touched set).""" + out: dict[tuple[str, str, tuple[float, float]], float | None] = {} + for (stream, axis), rows in state.rows.items(): + for row in rows: + out[(stream, axis, (round(row["start"], 6), round(row["end"], 6)))] = row.get("aggregated_uncertainty") + return out + + +def _mean_over(state: BeliefState, stream: str, axis: str, buckets: set | None) -> float | None: + vals = [] + for row in state.axis_rows(stream, axis): + if buckets is not None and (round(row["start"], 6), round(row["end"], 6)) not in buckets: + continue + u = row.get("aggregated_uncertainty") + if u is not None: + vals.append(float(u)) + return sum(vals) / len(vals) if vals else None + + +def _iteration_entry(cand: dict[str, Any], round_idx: int) -> dict[str, Any]: + return { + "intervention_id": cand.get("intervention_id") + or f"{round_idx}_{cand['rule']}_{cand.get('region_id') or 'global'}", + "round": round_idx, + "rule": cand["rule"], + "region_id": cand.get("region_id"), + "axis": cand.get("axis"), + "cost_class": cand.get("cost_class"), + "priority": cand.get("priority"), + "trigger": cand.get("trigger"), + "status": cand.get("status"), + "error": cand.get("error"), + } + + +def _write_round_belief(round_dir: Path, state: BeliefState, passes: list[str]) -> None: + import pandas as pd + + belief_dir = round_dir / "belief" + belief_dir.mkdir(parents=True, exist_ok=True) + for axis in AXES: + rows = [] + for stream in passes: + for r in state.axis_rows(stream, axis): + rows.append( + { + "stream": stream, + "start": r["start"], + "end": r["end"], + "aggregated_uncertainty": r.get("aggregated_uncertainty"), + "p_voice": r.get("p_voice"), + "epistemic": r.get("epistemic"), + "aleatoric_floor": r.get("aleatoric_floor"), + "status": r.get("status"), + "irreducible_reason": r.get("irreducible_reason"), + "round": r.get("round"), + "n_sources": len(r.get("contributing_sources") or []), + } + ) + if rows: + pd.DataFrame(rows).to_parquet(belief_dir / f"{axis}.parquet", index=False) + + +def _write_round_votes(round_dir: Path, store: VoteStore, round_idx: int) -> None: + import pandas as pd + + votes = store.votes_added_in_round(round_idx) + if votes: + pd.DataFrame([v.to_record() for v in votes]).to_parquet(round_dir / "votes_added.parquet", index=False) + + +def _json_safe_result(result: dict[str, Any]) -> dict[str, Any]: + return json.loads(json.dumps(result, default=str)) diff --git a/src/senselab/audio/workflows/audio_analysis/adaptive/ls_final.py b/src/senselab/audio/workflows/audio_analysis/adaptive/ls_final.py new file mode 100644 index 000000000..e4aabe3c3 --- /dev/null +++ b/src/senselab/audio/workflows/audio_analysis/adaptive/ls_final.py @@ -0,0 +1,263 @@ +"""Label Studio final tracks + resolved-disagreements index (FR-023, tasks.md T032). + +Additive only: reads the run's LS bundle, appends ``final__*`` tracks for the fusion +stream's task, and writes copies under ``/final/`` — the original bundle is +never modified. Also emits ``disagreements_resolved.json``: the run's round-1 +disagreements annotated with each bucket's final status and the interventions that +touched it — the before/after story of the loop. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +_CONF_BINS = (("high", 0.66), ("medium", 0.33), ("low", 0.0)) + + +def _conf_bin(c: float) -> str: + for name, lo in _CONF_BINS: + if c >= lo: + return name + return "low" + + +def _region( + rid: str, from_name: str, start: float, end: float, labels: list[str] | None = None, text: list[str] | None = None +) -> dict[str, Any]: + value: dict[str, Any] = {"start": round(start, 4), "end": round(end, 4)} + kind = "labels" + if labels is not None: + value["labels"] = labels + if text is not None: + value["text"] = text + kind = "textarea" + return {"id": rid, "from_name": from_name, "to_name": "audio", "type": kind, "value": value} + + +def _task_pass_label(task: dict[str, Any]) -> str | None: + """Infer the pass a LS task belongs to from its region-id prefixes.""" + for pred in task.get("predictions") or []: + for region in pred.get("result") or []: + from_name = str(region.get("from_name") or "") + if "__" in from_name: + return from_name.split("__", 1)[0] + return None + + +def build_final_ls_bundle( + *, + out_dir: Path, + run_dir: Path, + transcript: dict[str, Any], + diarization: dict[str, Any], + presence_rows: list[dict[str, Any]], + fusion_stream: str, + iterations: list[dict[str, Any]], +) -> dict[str, Any]: + """Write final/{labelstudio_tasks.json, labelstudio_config.xml, disagreements_resolved.json}.""" + final = Path(out_dir) / "final" + final.mkdir(parents=True, exist_ok=True) + report: dict[str, Any] = {} + + tasks_path = Path(run_dir) / "labelstudio_tasks.json" + config_path = Path(run_dir) / "labelstudio_config.xml" + if tasks_path.exists() and config_path.exists(): + tasks = json.loads(tasks_path.read_text()) + config = config_path.read_text() + + regions: list[dict[str, Any]] = [] + for i, w in enumerate(transcript.get("words") or []): + conf = float(w.get("confidence") or 0.0) + regions.append( + _region( + f"final_word_{i:04d}", + "final__consensus_transcript", + w["start"], + w["end"], + labels=[_conf_bin(conf)], + ) + ) + spk = f" [{w['speaker']}]" if w.get("speaker") else "" + regions.append( + _region( + f"final_word_{i:04d}", + "final__consensus_transcript__text", + w["start"], + w["end"], + text=[f"{w['text']} ({conf:.2f}){spk}"], + ) + ) + cluster_values = sorted({str(s["cluster_id"]) for s in diarization.get("segments") or []}) + for i, seg in enumerate(diarization.get("segments") or []): + regions.append( + _region( + f"final_diar_{i:04d}", + "final__diarization", + seg["start"], + seg["end"], + labels=[str(seg["cluster_id"])], + ) + ) + # Presence status runs (merge consecutive equal statuses). + run_start: float = 0.0 + run_status: str | None = None + prev_end = 0.0 + status_regions: list[tuple[float, float, str]] = [] + for row in presence_rows: + status = str(row.get("status") or "open") + if run_status is None: + run_start, run_status = row["start"], status + elif status != run_status or row["start"] > prev_end + 1e-6: + status_regions.append((run_start, prev_end, run_status)) + run_start, run_status = row["start"], status + prev_end = row["end"] + if run_status is not None: + status_regions.append((run_start, prev_end, run_status)) + for i, (s, e, status) in enumerate(status_regions): + regions.append(_region(f"final_presence_{i:04d}", "final__presence", s, e, labels=[status])) + + # Attach to the fusion stream's task (fallback: first task). + target = next((t for t in tasks if _task_pass_label(t) == fusion_stream), tasks[0] if tasks else None) + if target is not None: + preds = target.setdefault("predictions", [{"model_version": "adaptive_final", "score": 1.0, "result": []}]) + preds[0].setdefault("result", []).extend(regions) + + # Config: declare the three new tracks before the closing . + status_values = sorted({s for _, _, s in status_regions} | {"converged", "irreducible", "open"}) + blocks = [''] + blocks += [f' ", + '