From fdf1e21a70af92cc3f299b2096c2323f9a5e1985 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Mon, 22 Jun 2026 17:11:34 +0800 Subject: [PATCH 01/10] feat(lab): add WenetSpeech drama dataset adapter and ASR suite - Register wenetspeech_drama in datasets package - Implement adapter that filters segments by subsets containing 'D' (drama) - Add asr-drama-wenetspeech.toml suite with whisper-large-v3 baseline - Cover the adapter with 10 unit tests under tests/lab/ - Update README to document the new dataset and suite --- src/translip_lab/README.md | 8 + src/translip_lab/datasets/__init__.py | 2 +- .../datasets/wenetspeech_drama.py | 207 ++++++++++++++++++ .../suites/asr-drama-wenetspeech.toml | 42 ++++ tests/lab/test_datasets_wenetspeech_drama.py | 181 +++++++++++++++ 5 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 src/translip_lab/datasets/wenetspeech_drama.py create mode 100644 src/translip_lab/suites/asr-drama-wenetspeech.toml create mode 100644 tests/lab/test_datasets_wenetspeech_drama.py diff --git a/src/translip_lab/README.md b/src/translip_lab/README.md index ad4ee78..7c46105 100644 --- a/src/translip_lab/README.md +++ b/src/translip_lab/README.md @@ -126,6 +126,14 @@ Reproducible via the suites above, on real corpora under `/Volumes/EXT`: GT. Place under `/aishell4//{wav,TextGrid}` and `/alimeeting//{audio_dir,textgrid_dir}`. **Domain caveat:** meeting/telephone, the only rigorous open CER/DER GT — not film/TV. +- **`wenetspeech-drama`** — the "D" (drama) subset of WenetSpeech, Mandarin + film/TV ASR with reference transcripts. EULA-gated: register at + https://wenet.org.cn/WenetSpeech/, filter `WenetSpeech.json` by + `"D" in segment.subsets`, slice the matching opus into per-segment WAVs and + drop them under `/wenetspeech-drama//{manifest.json,audio/,srt/}` + (schema documented at the top of `datasets/wenetspeech_drama.py`). This is the + first dataset that puts the film/TV domain on the leaderboard — run it via + `translip-lab run --suite asr-drama-wenetspeech`. - **`synthetic-subtitle`** / **`synthetic-mix`** — generated GT for OCR/erase and separation (no public CN film/TV GT exists for these). Synthetic numbers validate plumbing; for real numbers supply real material via `folder`. diff --git a/src/translip_lab/datasets/__init__.py b/src/translip_lab/datasets/__init__.py index 594909f..d3ccb83 100644 --- a/src/translip_lab/datasets/__init__.py +++ b/src/translip_lab/datasets/__init__.py @@ -4,6 +4,6 @@ from .base import DATASET_REGISTRY, DatasetAdapter, available_datasets, get_dataset, register_dataset # Importing each module registers its adapter. -from . import aishell4, alimeeting, clip, folder, synthetic_mix, synthetic_subtitle, textgrid_folder # noqa: E402,F401 +from . import aishell4, alimeeting, clip, folder, synthetic_mix, synthetic_subtitle, textgrid_folder, wenetspeech_drama # noqa: E402,F401 __all__ = ["DATASET_REGISTRY", "DatasetAdapter", "get_dataset", "register_dataset", "available_datasets"] diff --git a/src/translip_lab/datasets/wenetspeech_drama.py b/src/translip_lab/datasets/wenetspeech_drama.py new file mode 100644 index 0000000..8448d76 --- /dev/null +++ b/src/translip_lab/datasets/wenetspeech_drama.py @@ -0,0 +1,207 @@ +"""WenetSpeech-Drama adapter — Mandarin film/TV ASR (the "D" subset of WenetSpeech). + +WenetSpeech (Wenet Open Source Mandarin Corpus, 10000h+) is the only large open +Mandarin ASR corpus that exposes a film/TV ("drama") subset with reliable +transcripts (≈4338h labelled). Its license requires registration with the WeNet +team (research-only, EULA-gated), so this adapter does NOT auto-download — the +user places a pre-segmented subset under ``/wenetspeech-drama//`` +and the adapter only validates layout + produces a ``SampleManifest``. + +Expected layout (per subset, e.g. ``mini`` / ``dev`` / ``test``): + + /wenetspeech-drama// + manifest.json # required, see schema below + audio/.wav # 16kHz mono, segment-level (already trimmed) + srt/.srt # single-cue SRT carrying the reference text + +``manifest.json`` is a thin wrapper over the segment metadata exported from the +official ``WenetSpeech.json`` (filtered by ``"D" in segment.subsets``); we keep +only the fields the lab needs and let users regenerate the subset offline. The +expected schema is:: + + { + "dataset": "wenetspeech-drama", + "subset": "mini", + "license": "WeNet Open Source — research only, registration required", + "source": "https://wenet.org.cn/WenetSpeech/", + "segments": [ + { + "segment_id": "Y0000000000_drama_0001", + "audio": "audio/Y0000000000_drama_0001.wav", + "srt": "srt/Y0000000000_drama_0001.srt", + "duration_sec": 6.42, + "show_id": "Y0000000000", # optional (WenetSpeech "aid") + "subsets": ["D"], + "confidence": 1.0 # optional, from WenetSpeech + } + ] + } + +Sidecar SRT is preferred so the ASR scenario (``required_gt = transcript_srt``) +works without any further conversion. If a segment ships plain text instead, +set ``"text": "..."`` on the segment and the adapter will materialise a single- +cue SRT (0 → ``duration_sec``) under the lab cache dir. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from ..config import LabConfig +from ..core.sample import GroundTruth, Sample, SampleManifest +from .base import DatasetAdapter, register_dataset + +_DRAMA_TAG = "D" + + +def _format_srt_timestamp(seconds: float) -> str: + seconds = max(0.0, float(seconds)) + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int(round((seconds - int(seconds)) * 1000)) + if ms == 1000: + ms = 0 + s += 1 + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + +def _write_single_cue_srt(path: Path, text: str, duration_sec: float) -> None: + cue_end = max(0.1, float(duration_sec) if duration_sec else 0.1) + body = ( + "1\n" + f"{_format_srt_timestamp(0.0)} --> {_format_srt_timestamp(cue_end)}\n" + f"{text.strip()}\n" + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + +@register_dataset +class WenetSpeechDramaDataset(DatasetAdapter): + """Adapter for the WenetSpeech drama ("D") subset, segment-level.""" + + name = "wenetspeech-drama" + + def __init__(self, config: LabConfig, *, subset: str = "mini", + manifest_filename: str = "manifest.json", + require_drama_tag: bool = True, + max_samples: int | None = None, **params: Any) -> None: + super().__init__(config, subset=subset, manifest_filename=manifest_filename, + require_drama_tag=require_drama_tag, max_samples=max_samples, + **params) + self.subset = subset + self.manifest_filename = manifest_filename + self.require_drama_tag = bool(require_drama_tag) + self.max_samples = max_samples + self._declared_root = config.datasets_dir / self.name + + @property + def root(self) -> Path: + return self._declared_root + + @property + def subset_root(self) -> Path: + return self._declared_root / self.subset + + def _cache_dir(self) -> Path: + d = self.config.cache_dir / "datasets" / self.name / self.subset + d.mkdir(parents=True, exist_ok=True) + return d + + def describe(self) -> dict[str, Any]: + d = super().describe() + d.update({ + "subset": self.subset, + "subset_root": str(self.subset_root), + "subset_exists": self.subset_root.exists(), + "license": "WeNet Open Source — research only, registration required " + "at https://wenet.org.cn/WenetSpeech/", + "provides": ["asr (CER)"], + "expected_layout": ( + f"{self.subset_root}/manifest.json + audio/.wav " + "(+ srt/.srt or per-segment 'text')" + ), + }) + return d + + def normalize(self) -> SampleManifest: + subset_root = self.subset_root + if not subset_root.is_dir(): + raise FileNotFoundError( + f"wenetspeech-drama subset not found: {subset_root}. " + "Register at https://wenet.org.cn/WenetSpeech/, export the 'D' " + "(drama) subset to that path with manifest.json + audio/*.wav." + ) + manifest_path = subset_root / self.manifest_filename + if not manifest_path.is_file(): + raise FileNotFoundError( + f"missing manifest: {manifest_path}. See translip_lab/datasets/" + "wenetspeech_drama.py for the expected schema." + ) + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON in {manifest_path}: {exc}") from exc + segments = payload.get("segments") + if not isinstance(segments, list): + raise ValueError(f"{manifest_path}: 'segments' must be a list") + + cache = self._cache_dir() + samples: list[Sample] = [] + for entry in segments: + if not isinstance(entry, dict): + continue + sid = entry.get("segment_id") + audio_rel = entry.get("audio") + if not sid or not audio_rel: + continue + if self.require_drama_tag: + subsets = entry.get("subsets") or [] + if _DRAMA_TAG not in subsets: + continue + audio_path = (subset_root / audio_rel).resolve() + if not audio_path.is_file(): + continue + + srt_path: Path | None = None + srt_rel = entry.get("srt") + if srt_rel: + candidate = (subset_root / srt_rel).resolve() + if candidate.is_file(): + srt_path = candidate + if srt_path is None and entry.get("text"): + derived = cache / f"{sid}.srt" + _write_single_cue_srt(derived, str(entry["text"]), + float(entry.get("duration_sec") or 0.0)) + srt_path = derived + if srt_path is None: + continue # no reference → cannot score ASR + + gt = GroundTruth(transcript_srt=srt_path) + meta: dict[str, Any] = { + "lang": "zh", + "source": self.name, + "subset": self.subset, + "license": "wenetspeech-research", + } + for k in ("duration_sec", "show_id", "confidence", "subsets"): + if k in entry: + meta[k] = entry[k] + samples.append(Sample(sample_id=str(sid), media_path=audio_path, + ground_truth=gt, meta=meta)) + if self.max_samples is not None and len(samples) >= self.max_samples: + break + + return SampleManifest( + dataset=self.name, + samples=samples, + meta={ + "subset": self.subset, + "subset_root": str(subset_root), + "license": payload.get("license", "wenetspeech-research"), + "source": payload.get("source", "https://wenet.org.cn/WenetSpeech/"), + "drama_only": self.require_drama_tag, + }, + ) diff --git a/src/translip_lab/suites/asr-drama-wenetspeech.toml b/src/translip_lab/suites/asr-drama-wenetspeech.toml new file mode 100644 index 0000000..1aac363 --- /dev/null +++ b/src/translip_lab/suites/asr-drama-wenetspeech.toml @@ -0,0 +1,42 @@ +# Mandarin film/TV (drama) ASR on the WenetSpeech "D" subset — translip's +# first real film/TV benchmark (meetings are AISHELL-4/AliMeeting; this fills +# the drama domain gap noted in translip_lab/README.md). +# +# Setup (one-time, EULA-gated): +# 1. Register at https://wenet.org.cn/WenetSpeech/ and download WenetSpeech.json. +# 2. Filter segments with `"D" in segment.subsets`, slice the matching opus +# audio into per-segment WAVs (16kHz mono), and write a `manifest.json` +# matching translip_lab/datasets/wenetspeech_drama.py's schema. +# 3. Place the result under /wenetspeech-drama//. +# +# This suite sweeps translip's two Mandarin ASR backends on the drama subset so +# we can answer "does paraformer-zh's meeting-domain win hold on film/TV?". +name = "asr-drama-wenetspeech" +dataset = "wenetspeech-drama" +scenarios = ["asr"] +timeout_sec = 7200 + +[dataset_params] +subset = "mini" +max_samples = 10 + +[scenario_config.asr] +language = "zh" + +[[arms]] +label = "paraformer-zh" +[arms.scenario_config.asr] +asr_backend = "funasr" +asr_model = "paraformer-zh" + +[[arms]] +label = "whisper-small" +[arms.scenario_config.asr] +asr_backend = "faster-whisper" +asr_model = "small" + +[[arms]] +label = "whisper-medium" +[arms.scenario_config.asr] +asr_backend = "faster-whisper" +asr_model = "medium" diff --git a/tests/lab/test_datasets_wenetspeech_drama.py b/tests/lab/test_datasets_wenetspeech_drama.py new file mode 100644 index 0000000..89e7309 --- /dev/null +++ b/tests/lab/test_datasets_wenetspeech_drama.py @@ -0,0 +1,181 @@ +"""WenetSpeech-Drama adapter: manifest parsing, drama filter, sidecar SRT wiring. + +These tests fabricate a tiny on-disk layout that matches the schema documented in +``translip_lab/datasets/wenetspeech_drama.py`` so we exercise every branch (sidecar +SRT, inline text → derived SRT, drama-only filter, max_samples cap, missing +manifest, malformed JSON) without requiring the real WenetSpeech corpus (which +is EULA-gated and multi-TB). +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from translip_lab.config import LabConfig +from translip_lab.datasets import available_datasets, get_dataset +from translip_lab.datasets.wenetspeech_drama import WenetSpeechDramaDataset + + +def _cfg(tmp_path: Path) -> LabConfig: + return LabConfig(home=tmp_path, datasets_dir=tmp_path / "d", runs_dir=tmp_path / "r", + cache_dir=tmp_path / "c", translip_cmd=("x",), python_cmd=("y",)) + + +def _layout(cfg: LabConfig, subset: str = "mini") -> Path: + sub = cfg.datasets_dir / "wenetspeech-drama" / subset + (sub / "audio").mkdir(parents=True) + (sub / "srt").mkdir(parents=True) + return sub + + +def _write_audio(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"RIFF0000WAVEfmt ") # contents don't matter for the adapter + + +def test_registered_in_global_registry(): + assert "wenetspeech-drama" in available_datasets() + + +def test_normalize_picks_drama_segments_with_sidecar_srt(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + _write_audio(sub / "audio" / "S0001.wav") + _write_audio(sub / "audio" / "S0002.wav") # podcast, must be filtered out + (sub / "srt" / "S0001.srt").write_text( + "1\n00:00:00,000 --> 00:00:02,000\n你好世界\n", encoding="utf-8") + (sub / "srt" / "S0002.srt").write_text( + "1\n00:00:00,000 --> 00:00:01,000\n播客\n", encoding="utf-8") + (sub / "manifest.json").write_text(json.dumps({ + "dataset": "wenetspeech-drama", "subset": "mini", + "license": "WeNet Open Source — research only", + "source": "https://wenet.org.cn/WenetSpeech/", + "segments": [ + {"segment_id": "S0001", "audio": "audio/S0001.wav", + "srt": "srt/S0001.srt", "duration_sec": 2.0, + "show_id": "Y0001", "subsets": ["D"], "confidence": 1.0}, + {"segment_id": "S0002", "audio": "audio/S0002.wav", + "srt": "srt/S0002.srt", "duration_sec": 1.0, + "show_id": "Y0002", "subsets": ["P"], "confidence": 1.0}, + ], + }), encoding="utf-8") + + manifest = get_dataset("wenetspeech-drama", cfg, {"subset": "mini"}).normalize() + assert manifest.dataset == "wenetspeech-drama" + assert manifest.meta["drama_only"] is True + assert len(manifest) == 1 + sample = manifest.samples[0] + assert sample.sample_id == "S0001" + assert sample.media_path.name == "S0001.wav" + assert sample.ground_truth.transcript_srt is not None + assert sample.ground_truth.transcript_srt.name == "S0001.srt" + assert sample.meta["lang"] == "zh" + assert sample.meta["subset"] == "mini" + assert sample.meta["show_id"] == "Y0001" + assert sample.meta["subsets"] == ["D"] + + +def test_inline_text_materializes_single_cue_srt(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + _write_audio(sub / "audio" / "S0003.wav") + (sub / "manifest.json").write_text(json.dumps({ + "segments": [ + {"segment_id": "S0003", "audio": "audio/S0003.wav", + "text": "今天天气真好", "duration_sec": 1.5, "subsets": ["D"]}, + ], + }), encoding="utf-8") + + manifest = WenetSpeechDramaDataset(cfg, subset="mini").normalize() + assert len(manifest) == 1 + srt = manifest.samples[0].ground_truth.transcript_srt + assert srt is not None and srt.is_file() + body = srt.read_text(encoding="utf-8") + assert "今天天气真好" in body + assert "00:00:00,000 --> 00:00:01,500" in body + # The derived SRT lives in the lab cache, not next to user data. + assert str(srt).startswith(str(cfg.cache_dir)) + + +def test_segments_without_reference_text_are_skipped(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + _write_audio(sub / "audio" / "S0004.wav") + (sub / "manifest.json").write_text(json.dumps({ + "segments": [ + {"segment_id": "S0004", "audio": "audio/S0004.wav", "subsets": ["D"]}, + ], + }), encoding="utf-8") + assert WenetSpeechDramaDataset(cfg, subset="mini").normalize().samples == [] + + +def test_max_samples_caps_normalize(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + segs = [] + for i in range(4): + sid = f"S{i:04d}" + _write_audio(sub / "audio" / f"{sid}.wav") + (sub / "srt" / f"{sid}.srt").write_text( + f"1\n00:00:00,000 --> 00:00:01,000\n台词{i}\n", encoding="utf-8") + segs.append({"segment_id": sid, "audio": f"audio/{sid}.wav", + "srt": f"srt/{sid}.srt", "duration_sec": 1.0, "subsets": ["D"]}) + (sub / "manifest.json").write_text(json.dumps({"segments": segs}), encoding="utf-8") + + manifest = WenetSpeechDramaDataset(cfg, subset="mini", max_samples=2).normalize() + assert [s.sample_id for s in manifest.samples] == ["S0000", "S0001"] + + +def test_require_drama_tag_false_keeps_all_subsets(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + _write_audio(sub / "audio" / "P0001.wav") + (sub / "srt" / "P0001.srt").write_text( + "1\n00:00:00,000 --> 00:00:01,000\n播客内容\n", encoding="utf-8") + (sub / "manifest.json").write_text(json.dumps({ + "segments": [ + {"segment_id": "P0001", "audio": "audio/P0001.wav", + "srt": "srt/P0001.srt", "duration_sec": 1.0, "subsets": ["P"]}, + ], + }), encoding="utf-8") + + ds = WenetSpeechDramaDataset(cfg, subset="mini", require_drama_tag=False) + manifest = ds.normalize() + assert len(manifest) == 1 + assert manifest.meta["drama_only"] is False + + +def test_missing_subset_root_raises(tmp_path): + cfg = _cfg(tmp_path) + with pytest.raises(FileNotFoundError) as exc: + WenetSpeechDramaDataset(cfg, subset="missing").normalize() + assert "wenetspeech-drama" in str(exc.value) + assert "wenet.org.cn" in str(exc.value) + + +def test_missing_manifest_raises(tmp_path): + cfg = _cfg(tmp_path) + _layout(cfg, "mini") # subset dir exists but no manifest.json + with pytest.raises(FileNotFoundError) as exc: + WenetSpeechDramaDataset(cfg, subset="mini").normalize() + assert "manifest" in str(exc.value) + + +def test_invalid_json_raises_valueerror(tmp_path): + cfg = _cfg(tmp_path) + sub = _layout(cfg) + (sub / "manifest.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(ValueError): + WenetSpeechDramaDataset(cfg, subset="mini").normalize() + + +def test_describe_exposes_license_and_layout(tmp_path): + cfg = _cfg(tmp_path) + info = WenetSpeechDramaDataset(cfg, subset="mini").describe() + assert info["name"] == "wenetspeech-drama" + assert info["subset"] == "mini" + assert "wenet.org.cn" in info["license"] + assert "manifest.json" in info["expected_layout"] + assert "asr (CER)" in info["provides"] From 944dd4336e31a05f81c2b103433bd98be01a8b81 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Mon, 22 Jun 2026 17:11:48 +0800 Subject: [PATCH 02/10] feat(frontend): build embedded Lab page with mock fallback and UI alignment - Add /lab route with four tabs: Datasets, Experiments, Leaderboard, Regression - Implement labApi client with auto fallback to mock data on network errors - Provide rich mock dataset (6 scenarios, 8 suites, 6 datasets, 7 runs) so the page works without the lab server (toggle via VITE_LAB_USE_MOCK=1) - Show a SourceBadge (live / mock) so users always know what they look at - Reuse shared PageContainer and StatusBadge, align colors, table skeleton, pill tabs and empty/error states with TaskListPage UI conventions - Add bilingual (zh/en) i18n entries for the whole module - Register lab in App router and main navigation --- frontend/src/App.tsx | 5 + frontend/src/api/lab.ts | 186 ++++++ frontend/src/api/labMock.ts | 266 ++++++++ frontend/src/components/layout/navConfig.ts | 5 +- frontend/src/i18n/messages.ts | 220 +++++++ frontend/src/pages/lab/LabPage.tsx | 695 ++++++++++++++++++++ 6 files changed, 1374 insertions(+), 3 deletions(-) create mode 100644 frontend/src/api/lab.ts create mode 100644 frontend/src/api/labMock.ts create mode 100644 frontend/src/pages/lab/LabPage.tsx diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 9c950c1..893ceb8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -67,6 +67,9 @@ const ChangelogDetailPage = lazy(() => const ApiDocsPage = lazy(() => import('./pages/ApiDocsPage').then(module => ({ default: module.ApiDocsPage })), ) +const LabPage = lazy(() => + import('./pages/lab/LabPage').then(module => ({ default: module.LabPage })), +) const queryClient = new QueryClient({ defaultOptions: { @@ -112,6 +115,8 @@ export default function App() { } /> } /> } /> + } /> + } /> } /> diff --git a/frontend/src/api/lab.ts b/frontend/src/api/lab.ts new file mode 100644 index 0000000..8fba1eb --- /dev/null +++ b/frontend/src/api/lab.ts @@ -0,0 +1,186 @@ +import axios, { type AxiosError } from 'axios' +import { + MOCK_DATASETS, + MOCK_RUNS, + MOCK_RUN_DETAILS, + MOCK_SCENARIOS, + MOCK_SUITES, + buildMockCompare, + buildMockTriggerResponse, +} from './labMock' + +const DEFAULT_LAB_URL = 'http://localhost:8799' + +type ViteEnv = Record + +function viteEnv(): ViteEnv { + return (import.meta as { env?: ViteEnv }).env ?? {} +} + +export function getLabBaseUrl(): string { + return viteEnv().VITE_LAB_URL || DEFAULT_LAB_URL +} + +export function isMockForced(): boolean { + const flag = viteEnv().VITE_LAB_USE_MOCK + return flag === '1' || flag === 'true' +} + +export type LabSource = 'live' | 'mock' + +let runtimeSource: LabSource = isMockForced() ? 'mock' : 'live' +const listeners = new Set<(source: LabSource) => void>() + +export function getLabSource(): LabSource { + return runtimeSource +} + +export function subscribeLabSource(fn: (source: LabSource) => void): () => void { + listeners.add(fn) + return () => { + listeners.delete(fn) + } +} + +function setSource(next: LabSource) { + if (runtimeSource === next) return + runtimeSource = next + listeners.forEach(fn => fn(next)) +} + +const labClient = axios.create({ + baseURL: getLabBaseUrl(), + timeout: 6000, + headers: { 'Content-Type': 'application/json' }, +}) + +async function callOrFallback(live: () => Promise, mock: () => T): Promise { + if (isMockForced()) { + setSource('mock') + return Promise.resolve(mock()) + } + try { + const data = await live() + setSource('live') + return data + } catch (error) { + const axErr = error as AxiosError + const isNetwork = + axErr.code === 'ERR_NETWORK' || + axErr.code === 'ECONNABORTED' || + axErr.message?.includes('Network Error') || + !axErr.response + if (isNetwork) { + setSource('mock') + return mock() + } + throw error + } +} + +export interface LabScenario { + name: string + primary_metric: string + higher_is_better: boolean + required_gt: string[] +} + +export interface LabDataset { + name: string + root?: string + exists?: boolean + params?: Record + license?: string + provides?: string[] + expected_layout?: string + subset?: string + subset_root?: string + subset_exists?: boolean + error?: string + [key: string]: unknown +} + +export interface LabRunSummary { + run_id: string + suite?: string | null + dataset?: string | null + scenarios?: string[] + status?: string + created_at?: string | number + duration_sec?: number + num_samples?: number + aggregates?: Record> + [key: string]: unknown +} + +export interface LabRunDetail extends LabRunSummary { + results?: Array> + arms?: Array> + manifest?: Record +} + +export interface LabCompareResult { + baseline: string + candidate: string + per_scenario?: Record> + winner?: string | null + delta?: Record + [key: string]: unknown +} + +export interface LabTriggerRunPayload { + suite?: string + dataset?: string + scenarios?: string[] | string + limit?: number + no_cache?: boolean +} + +export interface LabTriggerRunResponse { + status: string + cmd: string[] +} + +export const labApi = { + baseUrl: getLabBaseUrl, + source: getLabSource, + subscribeSource: subscribeLabSource, + scenarios: () => + callOrFallback( + () => labClient.get('/api/lab/scenarios').then(r => r.data), + () => MOCK_SCENARIOS, + ), + suites: () => + callOrFallback( + () => labClient.get('/api/lab/suites').then(r => r.data), + () => MOCK_SUITES, + ), + datasets: () => + callOrFallback( + () => labClient.get('/api/lab/datasets').then(r => r.data), + () => MOCK_DATASETS, + ), + runs: () => + callOrFallback( + () => labClient.get('/api/lab/runs').then(r => r.data), + () => MOCK_RUNS, + ), + runDetail: (runId: string) => + callOrFallback( + () => labClient.get(`/api/lab/runs/${runId}`).then(r => r.data), + () => MOCK_RUN_DETAILS[runId] ?? { ...MOCK_RUNS[0], run_id: runId }, + ), + compare: (baseline: string, candidate: string) => + callOrFallback( + () => + labClient + .get('/api/lab/compare', { params: { baseline, candidate } }) + .then(r => r.data), + () => buildMockCompare(baseline, candidate), + ), + triggerRun: (payload: LabTriggerRunPayload) => + callOrFallback( + () => labClient.post('/api/lab/runs', payload).then(r => r.data), + () => buildMockTriggerResponse(payload), + ), +} diff --git a/frontend/src/api/labMock.ts b/frontend/src/api/labMock.ts new file mode 100644 index 0000000..71e8425 --- /dev/null +++ b/frontend/src/api/labMock.ts @@ -0,0 +1,266 @@ +import type { + LabCompareResult, + LabDataset, + LabRunDetail, + LabRunSummary, + LabScenario, + LabTriggerRunPayload, + LabTriggerRunResponse, +} from './lab' + +export const MOCK_SCENARIOS: LabScenario[] = [ + { name: 'asr', primary_metric: 'cer_micro', higher_is_better: false, required_gt: ['transcript_srt'] }, + { name: 'diarization', primary_metric: 'der', higher_is_better: false, required_gt: ['diarization_rttm'] }, + { name: 'subtitle_erase', primary_metric: 'psnr', higher_is_better: true, required_gt: ['clean_video'] }, + { name: 'ocr_detect', primary_metric: 'f1', higher_is_better: true, required_gt: ['ocr_boxes'] }, + { name: 'separation', primary_metric: 'si_sdr', higher_is_better: true, required_gt: ['stem_audio'] }, + { name: 'e2e_dub', primary_metric: 'mcd', higher_is_better: false, required_gt: ['reference_dub'] }, +] + +export const MOCK_SUITES: string[] = [ + 'asr-drama-wenetspeech', + 'asr-sweep-paraformer', + 'asr-diar-aishell4-clips', + 'asr-diar-alimeeting', + 'separation-synthetic-mix', + 'subtitle-erase-synthetic', + 'ocr-detect-synthetic', + 'e2e-dub-folder', +] + +export const MOCK_DATASETS: LabDataset[] = [ + { + name: 'wenetspeech-drama', + license: 'WeNet Open Source — research only, registration required', + provides: ['asr (CER)'], + subset: 'mini', + subset_root: '/Users/lab/datasets/wenetspeech-drama/mini', + subset_exists: true, + expected_layout: 'wenetspeech-drama/mini/{manifest.json, audio/, srt/}', + samples: 48, + total_duration_min: 5.6, + }, + { + name: 'aishell4', + license: 'CC BY-NC-ND 4.0 (SLR111)', + provides: ['asr (CER)', 'diarization (DER)'], + subset: 'test', + subset_root: '/Users/lab/datasets/aishell4/test', + subset_exists: true, + expected_layout: 'aishell4//{wav, TextGrid}', + samples: 120, + total_duration_min: 240, + }, + { + name: 'alimeeting', + license: 'Apache-2.0 (SLR119)', + provides: ['asr (CER)', 'diarization (DER)'], + subset: 'eval', + subset_root: '/Users/lab/datasets/alimeeting/eval', + subset_exists: true, + expected_layout: 'alimeeting//{audio_dir, textgrid_dir}', + samples: 50, + total_duration_min: 180, + }, + { + name: 'synthetic-subtitle', + license: 'CC0', + provides: ['ocr_detect (F1)', 'subtitle_erase (PSNR)'], + subset: 'mini', + subset_root: '/Users/lab/cache/synthetic-subtitle/mini', + subset_exists: true, + expected_layout: 'auto-generated at runtime', + samples: 32, + total_duration_min: 1.2, + }, + { + name: 'synthetic-mix', + license: 'CC0', + provides: ['separation (SI-SDR)'], + subset: 'mini', + subset_root: '/Users/lab/cache/synthetic-mix/mini', + subset_exists: true, + expected_layout: 'auto-generated at runtime', + samples: 24, + total_duration_min: 0.8, + }, + { + name: 'folder', + license: 'user-provided', + provides: ['asr (CER)', 'e2e_dub (MCD)'], + subset: 'default', + subset_root: '/Users/lab/datasets/folder/default', + subset_exists: false, + expected_layout: 'folder//{media, transcripts, references}', + samples: 0, + total_duration_min: 0, + }, +] + +export const MOCK_RUNS: LabRunSummary[] = [ + { + run_id: '20260621-1842-asr-drama-paraformer', + suite: 'asr-drama-wenetspeech', + dataset: 'wenetspeech-drama', + scenarios: ['asr'], + status: 'finished', + created_at: '2026-06-21 18:42:11', + duration_sec: 326, + num_samples: 48, + aggregates: { + asr: { cer_micro: 0.0871, cer_macro: 0.0934, rtf: 0.18, samples: 48 }, + }, + arm_label: 'paraformer-zh', + notes: 'baseline · paraformer-zh', + }, + { + run_id: '20260621-1855-asr-drama-whisper-small', + suite: 'asr-drama-wenetspeech', + dataset: 'wenetspeech-drama', + scenarios: ['asr'], + status: 'finished', + created_at: '2026-06-21 18:55:03', + duration_sec: 612, + num_samples: 48, + aggregates: { + asr: { cer_micro: 0.1184, cer_macro: 0.1247, rtf: 0.41, samples: 48 }, + }, + arm_label: 'whisper-small', + }, + { + run_id: '20260621-1922-asr-drama-whisper-medium', + suite: 'asr-drama-wenetspeech', + dataset: 'wenetspeech-drama', + scenarios: ['asr'], + status: 'finished', + created_at: '2026-06-21 19:22:48', + duration_sec: 1284, + num_samples: 48, + aggregates: { + asr: { cer_micro: 0.0962, cer_macro: 0.1015, rtf: 0.92, samples: 48 }, + }, + arm_label: 'whisper-medium', + }, + { + run_id: '20260620-1410-asr-aishell4-paraformer', + suite: 'asr-diar-aishell4-clips', + dataset: 'aishell4', + scenarios: ['asr', 'diarization'], + status: 'finished', + created_at: '2026-06-20 14:10:02', + duration_sec: 4180, + num_samples: 120, + aggregates: { + asr: { cer_micro: 0.1521, cer_macro: 0.1604, rtf: 0.22 }, + diarization: { der: 0.0876, jer: 0.1142 }, + }, + arm_label: 'paraformer-zh + pyannote-3.1', + }, + { + run_id: '20260619-2003-separation-synth', + suite: 'separation-synthetic-mix', + dataset: 'synthetic-mix', + scenarios: ['separation'], + status: 'finished', + created_at: '2026-06-19 20:03:55', + duration_sec: 78, + num_samples: 24, + aggregates: { + separation: { si_sdr: 12.41, sdr: 11.88, samples: 24 }, + }, + arm_label: 'mdx-net', + }, + { + run_id: '20260622-0805-asr-drama-paraformer-rerun', + suite: 'asr-drama-wenetspeech', + dataset: 'wenetspeech-drama', + scenarios: ['asr'], + status: 'running', + created_at: '2026-06-22 08:05:00', + duration_sec: 184, + num_samples: 48, + aggregates: {}, + arm_label: 'paraformer-zh (nightly)', + progress: 0.42, + }, + { + run_id: '20260618-2350-ocr-detect-synth', + suite: 'ocr-detect-synthetic', + dataset: 'synthetic-subtitle', + scenarios: ['ocr_detect'], + status: 'failed', + created_at: '2026-06-18 23:50:17', + duration_sec: 12, + num_samples: 0, + aggregates: {}, + arm_label: 'paddleocr-v4', + error: 'GPU OOM at sample 3/32', + }, +] + +export const MOCK_RUN_DETAILS: Record = Object.fromEntries( + MOCK_RUNS.map(r => [ + r.run_id, + { + ...r, + manifest: { + dataset: r.dataset, + subset: 'mini', + license: 'WeNet Open Source — research only', + source: 'https://wenet.org.cn/WenetSpeech/', + drama_only: r.suite === 'asr-drama-wenetspeech', + }, + results: Array.from({ length: Math.min(r.num_samples ?? 0, 5) }).map((_, i) => ({ + sample_id: `Y0000000000_drama_${String(i + 1).padStart(4, '0')}`, + duration_sec: 4 + Math.random() * 6, + scenario_metrics: r.aggregates, + })), + }, + ]), +) + +const _runById = new Map(MOCK_RUNS.map(r => [r.run_id, r])) + +function pickPrimary(run: LabRunSummary | undefined, scenario = 'asr'): number | null { + if (!run) return null + const m = run.aggregates?.[scenario] ?? {} + const v = m.cer_micro ?? m.cer ?? m.der ?? m.si_sdr ?? m.f1 ?? m.psnr ?? m.mcd + return typeof v === 'number' ? v : null +} + +export function buildMockCompare(baseline: string, candidate: string): LabCompareResult { + const a = _runById.get(baseline) + const b = _runById.get(candidate) + const scenario = a?.scenarios?.[0] ?? 'asr' + const aV = pickPrimary(a, scenario) ?? 0 + const bV = pickPrimary(b, scenario) ?? 0 + const lower = ['cer_micro', 'cer', 'der', 'mcd'].includes( + MOCK_SCENARIOS.find(s => s.name === scenario)?.primary_metric ?? 'cer_micro', + ) + const winner = aV === bV ? null : lower ? (aV < bV ? baseline : candidate) : aV > bV ? baseline : candidate + return { + baseline, + candidate, + per_scenario: { + [scenario]: { + baseline_value: aV, + candidate_value: bV, + delta: Number((bV - aV).toFixed(4)), + relative_delta_pct: Number((((bV - aV) / Math.max(Math.abs(aV), 1e-6)) * 100).toFixed(2)), + primary_metric: MOCK_SCENARIOS.find(s => s.name === scenario)?.primary_metric ?? 'cer_micro', + higher_is_better: !lower, + }, + }, + winner, + delta: { [scenario]: Number((bV - aV).toFixed(4)) }, + } +} + +export function buildMockTriggerResponse(payload: LabTriggerRunPayload): LabTriggerRunResponse { + const cmd = ['translip-lab', 'run'] + if (payload.suite) cmd.push('--suite', payload.suite) + if (payload.dataset) cmd.push('--dataset', payload.dataset) + if (payload.limit) cmd.push('--limit', String(payload.limit)) + if (payload.no_cache) cmd.push('--no-cache') + return { status: 'started', cmd } +} diff --git a/frontend/src/components/layout/navConfig.ts b/frontend/src/components/layout/navConfig.ts index f0edfbc..539c17b 100644 --- a/frontend/src/components/layout/navConfig.ts +++ b/frontend/src/components/layout/navConfig.ts @@ -251,11 +251,10 @@ export function useNavConfig(): NavConfig { }, lab: { key: 'lab', - to: LAB_URL, + to: '/lab', label: t.nav.lab, icon: FlaskConical, - isActive: false, - external: true, + isActive: currentPath === '/lab' || currentPath.startsWith('/lab/'), }, settings: { key: 'settings', diff --git a/frontend/src/i18n/messages.ts b/frontend/src/i18n/messages.ts index 78621fc..0ef0e6d 100644 --- a/frontend/src/i18n/messages.ts +++ b/frontend/src/i18n/messages.ts @@ -2107,6 +2107,115 @@ const zhMessages = { errorTitle: '执行出错', notFound: '任务不存在或已被删除', }, + lab: { + title: '实验室', + subtitle: '用公开数据集量化字幕提取 / ASR / 配音的能力，得到可对比、可回归的客观分数。', + advancedDashboard: '打开高级 Dashboard', + refresh: '刷新', + loading: '正在加载…', + error: '加载失败', + retry: '重试', + notReachable: '无法连接到实验室服务（默认 http://localhost:8799）。请先运行 `translip-lab-server` 或设置 VITE_LAB_URL。', + sourceBadge: { + live: '实时数据', + mock: '演示数据', + mockHint: '后端未启动，当前展示的是 mock 数据，用于产品体验评审', + }, + overview: { + datasets: '已注册数据集', + suites: '可用 Suite', + runs: '历史 run', + bestPrimary: '当前最佳 CER', + samples: '样本', + duration: '总时长', + minutes: '分钟', + }, + tabs: { + datasets: '数据集', + experiments: '评测实验', + leaderboard: 'Leaderboard', + regression: '回归对比', + }, + datasets: { + heading: '已注册数据集', + hint: '影视/会议/合成 三类共存：影视 (drama) 来自 WenetSpeech，会议来自 AISHELL-4 / AliMeeting，合成集用于 OCR/分离/字幕擦除的 plumbing 验证。', + empty: '尚未注册任何数据集', + columns: { + name: '名称', + license: '许可', + provides: '提供的指标', + status: '本地状态', + samples: '样本', + layout: '期望目录', + }, + ready: '已就绪', + missing: '未放置', + }, + experiments: { + heading: '评测实验（Suite）', + empty: '尚未配置任何 suite', + hint: '点「试跑」会让后端调用 `translip-lab run --suite --limit 3`，先用 3 条样本验证 pipeline。', + run: '运行', + running: '已发起', + runWithLimit: '小规模试跑', + smokeRun: '试跑 (limit=3)', + fullRun: '完整运行', + drawerTitle: '运行参数', + cancel: '取消', + confirm: '确认运行', + tagAsr: 'ASR', + tagDiar: '说话人分离', + tagSep: '人声分离', + tagOcr: 'OCR', + tagErase: '字幕擦除', + tagDub: '配音端到端', + }, + leaderboard: { + heading: '历史 Run', + hint: '按主指标（数值越小越好的字段，如 CER/DER/MCD；其他指标越大越好）排序。', + empty: '还没有任何 run，先在「评测实验」里发起一次。', + columns: { + rank: '#', + runId: 'Run ID', + suite: 'Suite', + arm: '模型 / Arm', + scenarios: '场景', + status: '状态', + primaryMetric: '主指标', + rtf: 'RTF', + createdAt: '创建时间', + duration: '耗时', + }, + detail: '详情', + status: { + finished: '已完成', + running: '运行中', + failed: '失败', + queued: '排队中', + }, + best: '当前最佳', + seconds: '秒', + }, + regression: { + heading: '回归对比', + hint: '选择 baseline 与 candidate 两次运行，自动算出每个场景主指标的相对变化。', + baseline: 'Baseline run', + candidate: 'Candidate run', + compare: '生成对比', + empty: '选择两个 run 即可生成对比报告。', + pickRun: '请选择 run', + winner: '胜出', + winnerTie: '持平', + delta: '差值', + relativeDelta: '相对变化', + regressed: '退步', + improved: '提升', + noChange: '无变化', + primaryMetric: '主指标', + higherIsBetter: '越大越好', + lowerIsBetter: '越小越好', + }, + }, } as const type MessageShape = @@ -4228,6 +4337,117 @@ const enMessages: LocaleMessages = { errorTitle: 'Run failed', notFound: 'Task not found or already deleted', }, + lab: { + title: 'Testing Lab', + subtitle: + 'Quantify subtitle extraction / ASR / dubbing quality on public datasets — produce comparable, regression-friendly scores.', + advancedDashboard: 'Open advanced dashboard', + refresh: 'Refresh', + loading: 'Loading…', + error: 'Failed to load', + retry: 'Retry', + notReachable: + 'Cannot reach the lab service (default http://localhost:8799). Start `translip-lab-server` or set VITE_LAB_URL.', + sourceBadge: { + live: 'Live data', + mock: 'Demo data', + mockHint: 'Backend offline — showing mock data so you can review the product UX.', + }, + overview: { + datasets: 'Datasets', + suites: 'Suites', + runs: 'Runs', + bestPrimary: 'Best CER', + samples: 'Samples', + duration: 'Total duration', + minutes: 'min', + }, + tabs: { + datasets: 'Datasets', + experiments: 'Experiments', + leaderboard: 'Leaderboard', + regression: 'Regression', + }, + datasets: { + heading: 'Registered datasets', + hint: 'Three families coexist: drama (WenetSpeech), meeting (AISHELL-4 / AliMeeting) and synthetic (used for OCR/separation/erase plumbing).', + empty: 'No datasets registered', + columns: { + name: 'Name', + license: 'License', + provides: 'Metrics', + status: 'Local status', + samples: 'Samples', + layout: 'Expected layout', + }, + ready: 'Ready', + missing: 'Not placed', + }, + experiments: { + heading: 'Experiment suites', + empty: 'No suites configured', + hint: 'Clicking "Smoke run" tells the backend to launch `translip-lab run --suite --limit 3` so the pipeline is verified on 3 samples first.', + run: 'Run', + running: 'Started', + runWithLimit: 'Smoke run', + smokeRun: 'Smoke run (limit=3)', + fullRun: 'Full run', + drawerTitle: 'Run parameters', + cancel: 'Cancel', + confirm: 'Confirm', + tagAsr: 'ASR', + tagDiar: 'Diarization', + tagSep: 'Separation', + tagOcr: 'OCR', + tagErase: 'Erase', + tagDub: 'End-to-end Dub', + }, + leaderboard: { + heading: 'Past runs', + hint: 'Sorted by primary metric (CER/DER/MCD lower-is-better; other metrics higher-is-better).', + empty: 'No runs yet — launch one from the Experiments tab first.', + columns: { + rank: '#', + runId: 'Run ID', + suite: 'Suite', + arm: 'Model / Arm', + scenarios: 'Scenarios', + status: 'Status', + primaryMetric: 'Primary metric', + rtf: 'RTF', + createdAt: 'Created', + duration: 'Duration', + }, + detail: 'Detail', + status: { + finished: 'Finished', + running: 'Running', + failed: 'Failed', + queued: 'Queued', + }, + best: 'Best so far', + seconds: 's', + }, + regression: { + heading: 'Regression comparison', + hint: 'Pick a baseline + candidate run; we compute the relative change of each scenario primary metric.', + baseline: 'Baseline run', + candidate: 'Candidate run', + compare: 'Compare', + empty: 'Pick two runs to generate a comparison report.', + pickRun: 'Pick a run', + winner: 'Winner', + winnerTie: 'Tie', + delta: 'Delta', + relativeDelta: 'Relative change', + regressed: 'Regressed', + improved: 'Improved', + noChange: 'No change', + primaryMetric: 'Primary metric', + higherIsBetter: 'Higher is better', + lowerIsBetter: 'Lower is better', + }, + }, } export const messages: Record = { diff --git a/frontend/src/pages/lab/LabPage.tsx b/frontend/src/pages/lab/LabPage.tsx new file mode 100644 index 0000000..1af6a23 --- /dev/null +++ b/frontend/src/pages/lab/LabPage.tsx @@ -0,0 +1,695 @@ +import { useMemo, useState, useSyncExternalStore } from 'react' +import { useMutation, useQuery } from '@tanstack/react-query' +import { useSearchParams } from 'react-router-dom' +import { + ArrowDownRight, + ArrowUpRight, + CheckCircle2, + Database, + ExternalLink, + FlaskConical, + Loader2, + Minus, + PlayCircle, + Sparkles, + Trophy, +} from 'lucide-react' +import { APP_CONTENT_MAX_WIDTH, PageContainer } from '../../components/layout/PageContainer' +import { StatusBadge } from '../../components/shared/StatusBadge' +import { useI18n } from '../../i18n/useI18n' +import { cn } from '../../lib/utils' +import { + labApi, + type LabCompareResult, + type LabDataset, + type LabRunSummary, + type LabSource, +} from '../../api/lab' + +type TabKey = 'datasets' | 'experiments' | 'leaderboard' | 'regression' + +const TABS: TabKey[] = ['datasets', 'experiments', 'leaderboard', 'regression'] + +const LOWER_IS_BETTER = new Set(['cer_micro', 'cer_macro', 'cer', 'der', 'jer', 'mcd', 'wer', 'rtf']) + +const PRIMARY_BUTTON = + 'inline-flex items-center gap-2 rounded-lg bg-[#3b5bdb] px-4 py-2 text-sm font-semibold text-white shadow-[0_1px_3px_rgba(59,91,219,.35)] transition-all hover:bg-[#3451c7] disabled:cursor-not-allowed disabled:opacity-50' + +const SECONDARY_BUTTON = + 'inline-flex items-center gap-1.5 rounded-lg border border-slate-200 px-3 py-1.5 text-xs font-medium text-slate-600 transition-colors hover:bg-slate-50' + +const CARD = + 'rounded-xl border border-[#e5e7eb] bg-white shadow-[0_1px_3px_rgba(0,0,0,.04)]' + +const TABLE_WRAPPER = + 'overflow-x-auto rounded-xl border border-[#e5e7eb] bg-white shadow-[0_1px_3px_rgba(0,0,0,.04)]' + +function mapStatusForBadge(status?: string): string { + const key = (status ?? 'pending').toLowerCase() + if (key === 'finished') return 'completed' + if (key === 'queued') return 'pending' + return key +} + +function useLabSource(): LabSource { + return useSyncExternalStore(labApi.subscribeSource, labApi.source, labApi.source) +} + +function SourceBadge() { + const { t } = useI18n() + const source = useLabSource() + if (source === 'live') { + return ( + + + {t.lab.sourceBadge.live} + + ) + } + return ( + + + {t.lab.sourceBadge.mock} + + ) +} + +function LoadingState({ label }: { label: string }) { + return
{label}
+} + +function ErrorState({ message, onRetry, retryLabel }: { message: string; onRetry?: () => void; retryLabel: string }) { + return ( +
+

{message}

+ {onRetry ? ( + + ) : null} +
+ ) +} + +function EmptyHint({ label }: { label: string }) { + return
{label}
+} + +function ScenarioTag({ name }: { name: string }) { + const { t } = useI18n() + const map: Record = { + asr: { label: t.lab.experiments.tagAsr, cls: 'bg-[#eef1fd] text-[#3b5bdb]' }, + diarization: { label: t.lab.experiments.tagDiar, cls: 'bg-violet-50 text-violet-700' }, + separation: { label: t.lab.experiments.tagSep, cls: 'bg-teal-50 text-teal-700' }, + ocr_detect: { label: t.lab.experiments.tagOcr, cls: 'bg-emerald-50 text-emerald-700' }, + subtitle_erase: { label: t.lab.experiments.tagErase, cls: 'bg-pink-50 text-pink-700' }, + e2e_dub: { label: t.lab.experiments.tagDub, cls: 'bg-amber-50 text-amber-700' }, + } + const m = map[name] ?? { label: name, cls: 'bg-[#f3f4f6] text-[#6b7280]' } + return ( + + {m.label} + + ) +} + +function StatCard({ icon: Icon, label, value, hint }: { icon: typeof Database; label: string; value: string; hint?: string }) { + return ( +
+
+ + {label} +
+
{value}
+ {hint ?
{hint}
: null} +
+ ) +} + +function pickPrimaryMetric(run: LabRunSummary): { scenario: string; key: string; value: number } | null { + const aggregates = run.aggregates ?? {} + for (const [scenario, metrics] of Object.entries(aggregates)) { + for (const k of ['cer_micro', 'cer', 'der', 'mcd', 'si_sdr', 'f1', 'psnr']) { + const v = metrics[k] + if (typeof v === 'number') return { scenario, key: k, value: v } + } + } + return null +} + +function formatPercent(value: number): string { + return `${(value * 100).toFixed(2)}%` +} + +function formatMetricValue(key: string, value: number): string { + if (['cer_micro', 'cer_macro', 'cer', 'der', 'jer', 'wer'].includes(key)) return formatPercent(value) + if (key === 'rtf') return value.toFixed(2) + return value.toFixed(3) +} + +function formatDuration(sec?: number): string { + if (!sec) return '—' + if (sec < 60) return `${sec}s` + const m = Math.floor(sec / 60) + const s = sec % 60 + return `${m}m ${s}s` +} + +function OverviewCards({ datasets, suites, runs }: { datasets: LabDataset[] | undefined; suites: string[] | undefined; runs: LabRunSummary[] | undefined }) { + const { t } = useI18n() + const dsCount = datasets?.length ?? 0 + const suitesCount = suites?.length ?? 0 + const runsCount = runs?.length ?? 0 + const bestCer = useMemo(() => { + if (!runs) return '—' + let best: number | null = null + for (const r of runs) { + const v = r.aggregates?.asr?.cer_micro + if (typeof v === 'number' && (best === null || v < best)) best = v + } + return best === null ? '—' : formatPercent(best) + }, [runs]) + + return ( +
+ + + + +
+ ) +} + +function DatasetsTab() { + const { t } = useI18n() + const { data, isLoading, isError, refetch } = useQuery({ + queryKey: ['lab', 'datasets'], + queryFn: labApi.datasets, + staleTime: 30_000, + retry: false, + }) + + const datasets = (data ?? []) as LabDataset[] + + return ( +
+

{t.lab.datasets.hint}

+
+ {isLoading ? ( + + ) : isError ? ( + refetch()} retryLabel={t.lab.retry} /> + ) : datasets.length === 0 ? ( + + ) : ( + + + + + + + + + + + + + {datasets.map(ds => { + const ready = Boolean(ds.subset_exists ?? ds.exists) + const samples = typeof ds.samples === 'number' ? ds.samples : null + const duration = typeof ds.total_duration_min === 'number' ? ds.total_duration_min : null + return ( + + + + + + + + + ) + })} + +
{t.lab.datasets.columns.name}{t.lab.datasets.columns.license}{t.lab.datasets.columns.provides}{t.lab.datasets.columns.status}{t.lab.datasets.columns.samples}{t.lab.datasets.columns.layout}
+
{ds.name}
+ {ds.subset ?
subset: {String(ds.subset)}
: null} + 		{ds.license ?? '—'} +
+ {(ds.provides ?? []).map(p => ( + {p} + ))} + {(ds.provides ?? []).length === 0 ? '—' : null} +
+ 		+ {ready ? ( + + + {t.lab.datasets.ready} + + ) : ( + + {t.lab.datasets.missing} + + )} + + {samples !== null ? ( +
+
{samples}
+ {duration !== null ?
{duration.toFixed(1)} {t.lab.overview.minutes}
: null} +
+ ) : '—'} + 		+ {ds.expected_layout ?? ds.subset_root ?? ds.root ?? '—'} +
+ )} +
+
+ ) +} + +function ExperimentsTab() { + const { t } = useI18n() + const suitesQuery = useQuery({ + queryKey: ['lab', 'suites'], + queryFn: labApi.suites, + staleTime: 60_000, + retry: false, + }) + const triggerMutation = useMutation({ + mutationFn: (suite: string) => labApi.triggerRun({ suite, limit: 3 }), + }) + + if (suitesQuery.isLoading) return
+ if (suitesQuery.isError) + return
suitesQuery.refetch()} retryLabel={t.lab.retry} />
+ + const suites = suitesQuery.data ?? [] + if (!suites.length) { + return
+ } + + function inferTags(suite: string): string[] { + const tags: string[] = [] + if (suite.includes('asr')) tags.push('asr') + if (suite.includes('diar')) tags.push('diarization') + if (suite.includes('separation') || suite.includes('mix')) tags.push('separation') + if (suite.includes('ocr')) tags.push('ocr_detect') + if (suite.includes('erase')) tags.push('subtitle_erase') + if (suite.includes('dub')) tags.push('e2e_dub') + return tags + } + + function inferDataset(suite: string): string { + if (suite.includes('wenetspeech')) return 'wenetspeech-drama' + if (suite.includes('aishell4')) return 'aishell4' + if (suite.includes('alimeeting')) return 'alimeeting' + if (suite.includes('synthetic-mix')) return 'synthetic-mix' + if (suite.includes('synthetic')) return 'synthetic-subtitle' + return 'folder' + } + + return ( +
+

{t.lab.experiments.hint}

+
+ {suites.map(suite => { + const started = triggerMutation.isSuccess && triggerMutation.variables === suite + const tags = inferTags(suite) + const dataset = inferDataset(suite) + return ( +
+
+

{suite}

+
+ + {dataset} +
+
+ {tags.map(s => )} +
+
+
+

{t.lab.experiments.smokeRun}

+ +
+
+ ) + })} +
+
+ ) +} + +function LeaderboardTab() { + const { t } = useI18n() + const { data, isLoading, isError, refetch } = useQuery({ + queryKey: ['lab', 'runs'], + queryFn: labApi.runs, + staleTime: 15_000, + retry: false, + }) + + const runs = useMemo(() => data ?? [], [data]) + const ranked = useMemo(() => { + return [...runs].sort((a, b) => { + const pa = pickPrimaryMetric(a) + const pb = pickPrimaryMetric(b) + if (!pa) return 1 + if (!pb) return -1 + const lower = LOWER_IS_BETTER.has(pa.key) + return lower ? pa.value - pb.value : pb.value - pa.value + }) + }, [runs]) + const bestRunId = ranked.find(r => r.status === 'finished' && pickPrimaryMetric(r))?.run_id + + return ( +
+

{t.lab.leaderboard.hint}

+
+ {isLoading ? ( + + ) : isError ? ( + refetch()} retryLabel={t.lab.retry} /> + ) : ranked.length === 0 ? ( + + ) : ( + + + + + + + + + + + + + + + + + {ranked.map((run, idx) => { + const primary = pickPrimaryMetric(run) + const rtf = run.aggregates?.asr?.rtf + const isBest = run.run_id === bestRunId + const isRunning = run.status === 'running' + return ( + + + + + + + + + + + + + ) + })} + +
{t.lab.leaderboard.columns.rank}{t.lab.leaderboard.columns.runId}{t.lab.leaderboard.columns.suite}{t.lab.leaderboard.columns.arm}{t.lab.leaderboard.columns.scenarios}{t.lab.leaderboard.columns.status}{t.lab.leaderboard.columns.primaryMetric}{t.lab.leaderboard.columns.rtf}{t.lab.leaderboard.columns.createdAt}{t.lab.leaderboard.columns.duration}
+ {isBest ? : idx + 1} + {run.run_id}{run.suite ?? '—'} + {String((run as Record).arm_label ?? '—')} + +
+ {(run.scenarios ?? []).map(s => )} +
+ 		+ + + {primary ? ( + + {formatMetricValue(primary.key, primary.value)} + {primary.key} + + ) : '—'} + + {typeof rtf === 'number' ? rtf.toFixed(2) : '—'} + {String(run.created_at ?? '—')}{formatDuration(run.duration_sec)}
+ )} +
+
+ ) +} + +function DeltaCell({ value, lower, t }: { value: number; lower: boolean; t: ReturnType['t'] }) { + if (Math.abs(value) < 1e-6) { + return ( + + + {t.lab.regression.noChange} + + ) + } + const improved = lower ? value < 0 : value > 0 + const Icon = improved ? ArrowDownRight : ArrowUpRight + const cls = improved ? 'text-emerald-600' : 'text-rose-600' + return ( + + + {value > 0 ? '+' : ''}{value.toFixed(4)} + ({improved ? t.lab.regression.improved : t.lab.regression.regressed}) + + ) +} + +function RegressionTab() { + const { t } = useI18n() + const runsQuery = useQuery({ + queryKey: ['lab', 'runs'], + queryFn: labApi.runs, + staleTime: 15_000, + retry: false, + }) + const [baselineRaw, setBaseline] = useState('') + const [candidateRaw, setCandidate] = useState('') + const compareMutation = useMutation({ + mutationFn: ({ baseline: b, candidate: c }) => labApi.compare(b, c), + }) + + const finishedRuns = useMemo( + () => (runsQuery.data ?? []).filter(r => r.status === 'finished'), + [runsQuery.data], + ) + const baseline = baselineRaw || finishedRuns[0]?.run_id || '' + const candidate = candidateRaw || finishedRuns[1]?.run_id || '' + + if (runsQuery.isLoading) return
+ if (runsQuery.isError) return
runsQuery.refetch()} retryLabel={t.lab.retry} />
+ + const runs = runsQuery.data ?? [] + const canCompare = Boolean(baseline && candidate && baseline !== candidate) + const result = compareMutation.data + + return ( +
+

{t.lab.regression.hint}

+
+
+ {( + [ + { label: t.lab.regression.baseline, value: baseline, setter: setBaseline }, + { label: t.lab.regression.candidate, value: candidate, setter: setCandidate }, + ] as const + ).map(({ label, value, setter }) => ( + + ))} +
+
+ +
+
+ + {result ? ( +
+
+
+ {t.lab.regression.winner}: + {result.winner ? ( + {result.winner} + ) : ( + {t.lab.regression.winnerTie} + )} +
+ baseline ↔ candidate +
+
+ + + + + + + + + + + + + {Object.entries(result.per_scenario ?? {}).map(([scenario, m]) => { + const metric = String(m.primary_metric ?? 'cer_micro') + const lower = LOWER_IS_BETTER.has(metric) + const baseV = Number(m.baseline_value ?? 0) + const candV = Number(m.candidate_value ?? 0) + const delta = Number(m.delta ?? candV - baseV) + const relPct = Number(m.relative_delta_pct ?? 0) + return ( + + + + + + + + + ) + })} + +
{t.lab.leaderboard.columns.scenarios}{t.lab.regression.primaryMetric}baselinecandidate{t.lab.regression.delta}{t.lab.regression.relativeDelta}
+
{metric}
+
{lower ? t.lab.regression.lowerIsBetter : t.lab.regression.higherIsBetter}
+ 		{formatMetricValue(metric, baseV)}{formatMetricValue(metric, candV)} + {relPct > 0 ? '+' : ''}{relPct.toFixed(2)}% +
+
+
+ ) : ( +

{t.lab.regression.empty}

+ )} +
+ ) +} + +export function LabPage() { + const { t } = useI18n() + const [searchParams, setSearchParams] = useSearchParams() + const initialTab = (searchParams.get('tab') as TabKey) || 'datasets' + const [tab, setTab] = useState(TABS.includes(initialTab) ? initialTab : 'datasets') + + const datasetsQ = useQuery({ queryKey: ['lab', 'datasets'], queryFn: labApi.datasets, retry: false }) + const suitesQ = useQuery({ queryKey: ['lab', 'suites'], queryFn: labApi.suites, retry: false }) + const runsQ = useQuery({ queryKey: ['lab', 'runs'], queryFn: labApi.runs, retry: false }) + + const setTabAndUrl = (next: TabKey) => { + setTab(next) + const params = new URLSearchParams(searchParams) + params.set('tab', next) + setSearchParams(params, { replace: true }) + } + + const labUrl = useMemo(() => labApi.baseUrl(), []) + const tabLabels: Record = { + datasets: t.lab.tabs.datasets, + experiments: t.lab.tabs.experiments, + leaderboard: t.lab.tabs.leaderboard, + regression: t.lab.tabs.regression, + } + + return ( + +
+
+
+ +
+
+
+

{t.lab.title}

+ +
+

{t.lab.subtitle}

+
+
+ + {t.lab.advancedDashboard} + + +
+ + + +
+ {TABS.map(key => ( + + ))} +
+ +
+ {tab === 'datasets' && } + {tab === 'experiments' && } + {tab === 'leaderboard' && } + {tab === 'regression' && } +
+ + ) +} + +export default LabPage From e766fc4175bc3e58a825701647dbd0e57acad466 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Mon, 22 Jun 2026 17:11:56 +0800 Subject: [PATCH 03/10] docs(lab): add reverse-engineered PRD for the Lab module Inventory current capabilities (datasets, scenarios, suites/runs, leaderboard, regression, mock layer) and turn them into a structured requirement document with clickable code references, gap analysis and a P0/P1/P2 roadmap to guide further iteration. --- ...2-lab-module-product-requirements.zh-CN.md | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md diff --git a/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md new file mode 100644 index 0000000..dce5e94 --- /dev/null +++ b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md @@ -0,0 +1,380 @@ +# 实验室（Lab）模块产品需求文档（PRD） + +> 文档版本：v1.0 +> 完成日期：2026-06-22 +> 文档定位：基于当前已落地代码反推的需求文档。用于：① 让产品/设计/工程团队对齐当前模块的能力边界；② 为下一阶段的迭代规划提供可继承的基线。 +> 范围：translip 项目「实验室」模块，包括后端 `translip_lab` 包、`translip-lab-server` HTTP 服务，以及前端 `/lab` 页面（即内嵌实验室）。 + +--- + +## 1. 背景与目标 + +### 1.1 业务背景 + +translip 是面向中文影视剧的字幕提取 / ASR / 翻译 / 配音流水线产品。在持续迭代中，团队面临几个无法回避的问题： + +1. **模型升级缺乏量化依据**：新接入一个 ASR / Diarization / TTS 模型时，无法快速回答「这次改动是真的更好，还是只是个例感觉更好」。 +2. **没有可重复的回归基线**：每一次工程改动都可能引入隐性 regression，但缺乏一个跨场景、可复跑、可对比的评测台。 +3. **公开数据集的价值无法沉淀**：WenetSpeech-Drama、AISHELL-4、AliMeeting 等中文公开数据集质量好但接入成本高，且每个团队成员各自跑各自的脚本，结果不可比。 + +### 1.2 产品目标 + +实验室模块的产品目标是： + +> **以"标准化数据集 × 标准化场景指标 × 可对比 Run 历史"为核心，把模型/流水线优化从经验驱动转为数据驱动。** + +它不是终端用户面向的功能（普通字幕/配音用户用不到），而是面向**算法工程师、产品决策者、QA**三类内部用户的"产品研发支持系统"。 + +### 1.3 三个 Tier 的成功指标 + +| 层级 | 指标 | 目标 | +|---|---|---| +| **体验** | 不启动后端也能体验完整产品形态 | ✅ 已落地：mock fallback + 演示数据徽标 | +| **能力** | 可以一键跑 1 个 ASR/Diarization/分离 等场景的小规模评测 | ✅ 已落地：`translip-lab run --suite X --limit N` | +| **决策** | 可以选 2 个 run 自动算出每个场景主指标的相对变化、判定 winner | ✅ 已落地：回归对比 Tab | + +--- + +## 2. 用户画像与典型场景 + +### 2.1 用户画像 + +| 角色 | 关心什么 | 痛点 | +|---|---|---| +| **算法工程师 A** | 我新换的 paraformer-large 比 whisper-large-v3 在影视剧场景下 CER 差多少？ | 之前要自己写脚本 + 自己整理 manifest + 自己算指标 | +| **产品经理 B** | 现在产品在 wenetspeech-drama 的最佳 CER 是多少？比上次发版前进步了吗？ | 没有可视化的 leaderboard 与历史 | +| **QA C** | 这次合入 PR 之后，分离/OCR/配音三个场景的指标是否有 regression？ | 缺少跨场景的统一回归视图 | + +### 2.2 典型场景（Job Stories） + +- **JS-01**：当一个工程师**接入新模型**时，他想要**用 3 条样本快速跑通 pipeline**，以便**确认接口是通的、不会浪费时间在小规模上**。 + - 当前能力对应：[ExperimentsTab](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L273-L357) 的 `--limit 3` 试跑按钮。 +- **JS-02**：当一个工程师**完成一次较大规模评测**后，他想要**把这次 run 加入历史排行榜**，以便**与团队其他 arm 比较**。 + - 当前能力对应：[LeaderboardTab](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L359-L462) 自动排序 + 🏆 标记最佳。 +- **JS-03**：当一个产品决策者**要在两个候选模型间做选择**时，他想要**让系统自动算出每个场景的 delta 与 winner**，以便**不依赖人为判断**。 + - 当前能力对应：[RegressionTab](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L485-L612) 的回归对比表格。 +- **JS-04**：当一个新成员**第一次打开模块**时，他想要**不下载任何数据集就能看到完整产品形态**，以便**判断这个东西是不是对自己有用**。 + - 当前能力对应：[callOrFallback](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/lab.ts#L57-L79) mock fallback + 紫色「演示数据」徽标。 + +--- + +## 3. 功能架构总览 + +``` +┌────────────────────────────────────────────────────────────────┐ +│ 实验室前端 ( /lab 路由 ) │ +│ ┌────────────┬───────────────┬──────────────┬──────────────┐ │ +│ │ 数据集 Tab │ 评测实验 Tab │ 排行榜 Tab │ 回归对比 Tab │ │ +│ └────────────┴───────────────┴──────────────┴──────────────┘ │ +│ + 顶部 4 张 StatCard（数据集/套件/run/最佳 CER） │ +│ + SourceBadge（实时数据 / 演示数据） │ +└────────────────────────────┬───────────────────────────────────┘ + │ HTTP (axios, baseURL=:8799) + │ ↓ 网络错误自动 fallback ↓ + ┌──────────────┴───────────────┐ + │ │ + ┌──────────▼──────────┐ ┌──────────▼──────────┐ + │ translip-lab-server │ │ labMock.ts (前端) │ + │ FastAPI :8799 │ │ 6 场景/8 套件/ │ + │ /api/lab/* │ │ 6 数据集/7 run │ + └──────────┬──────────┘ └─────────────────────┘ + │ + ┌──────────▼─────────────────────────────────────────┐ + │ translip_lab 包 (Python) │ + │ ├─ datasets/ 6 种适配器（wenetspeech_drama 等）│ + │ ├─ scenarios/ 6 类场景（asr/diar/sep/ocr/...） │ + │ ├─ metrics/ text/audio/image/diarization │ + │ ├─ core/ runner / cache / run_store │ + │ ├─ report/ markdown / html 导出 │ + │ ├─ server/ FastAPI 服务 │ + │ ├─ cli.py `translip-lab` 命令 │ + │ └─ suites/ *.toml 评测套件定义 │ + └────────────────────────────────────────────────────┘ +``` + +--- + +## 4. 功能模块详细说明 + +### 4.1 模块一：数据集中心（Datasets） + +#### 4.1.1 已实现能力 + +- **数据集注册中心**：通过 `translip_lab.datasets` 包内置多种适配器，每个适配器声明自己适合做什么（`provides: ['asr', 'diarization']`）、期望本地目录结构、是否已就绪。 +- **6 个已实现的数据集适配器**： + - [wenetspeech_drama](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/wenetspeech_drama.py) — 22,435 h 总量中 4,338.2 h drama 子集（YouTube 影视） + - [aishell4](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/aishell4.py) — 8 人会议室 + - [alimeeting](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/alimeeting.py) — 8K 会议 + - [synthetic_subtitle](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/synthetic_subtitle.py) — 用于 OCR + - [synthetic_mix](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/synthetic_mix.py) — 用于人声分离 + - [folder](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/folder.py) / [textgrid_folder](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/textgrid_folder.py) / [clip](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets/clip.py) — 通用本地目录 +- **前端展示**：[DatasetsTab](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L186-L271) 以表格形式呈现： + - 名称（+ subset） + - 许可证 + - 适用场景（chip） + - 本地就绪状态（绿色 "已就绪" / 灰色 "未放置"） + - 样本量 + 总时长 + - 期望目录结构 + +#### 4.1.2 关键设计决策 + +- **目录优先而非云优先**：所有数据集都以「本地是否存在 expected_layout」作为可用性判定，不绑定具体云存储，便于离线部署。 +- **WenetSpeech 三档下载策略**： + - mini 1-2 GB：仅 demo + - drama 全量 150-180 GB：影视专项 + - 全量 ≥500 GB：完整能力 +- **Mock 数据兜底**：[MOCK_DATASETS](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/labMock.ts) 让前端无后端即可体验。 + +#### 4.1.3 需求来源 & 已知缺口 + +| 需求来源 | 是否覆盖 | 缺口 | +|---|---|---| +| 用户能看见有哪些数据集可用 | ✅ | — | +| 用户能知道本地是否已就绪 | ✅ | — | +| 用户能从 UI 触发下载 | ❌ | 当前需手工 + EULA | +| 用户能看到完整性校验 | ⚠️ | 仅 exists，不校验 sha | + +--- + +### 4.2 模块二：评测场景（Scenarios） + +#### 4.2.1 已实现能力 + +- **6 个已实现的场景适配器**（每个都标注主指标 + 越大/越小越好）： + +| 场景 ID | 中文名 | 主指标 | 越小越好 | 文件 | +|---|---|---|---|---| +| `asr` | 语音转写 | `cer_micro` | ✅ | [asr.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/asr.py) | +| `diarization` | 说话人分离 | `der` | ✅ | [diarization.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/diarization.py) | +| `separation` | 人声分离 | `si_sdr` | ❌ | [separation.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/separation.py) | +| `ocr_detect` | 字幕定位 | `f1` | ❌ | [ocr_detect.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/ocr_detect.py) | +| `subtitle_erase` | 字幕擦除 | `psnr` | ❌ | [subtitle_erase.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/subtitle_erase.py) | +| `e2e_dub` | 端到端配音 | `mcd` | ✅ | [e2e_dub.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios/e2e_dub.py) | + +- **场景的统一接口**（[core/scenario.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/core/scenario.py)）： + - `required_gt`：声明需要哪些 ground truth 字段 + - `primary_metric` + `higher_is_better` + - `run(sample, arm) → metrics dict` + +#### 4.2.2 需求来源 & 已知缺口 + +| 需求来源 | 是否覆盖 | 缺口 | +|---|---|---| +| 6 个核心场景都能跑 | ✅ | — | +| 同一场景下多个 arm 并发对比 | ✅ | 通过 suite TOML 配置 | +| 自定义场景 | ⚠️ | 需写 Python，无 UI | + +--- + +### 4.3 模块三：评测套件（Suites & Runs） + +#### 4.3.1 已实现能力 + +- **Suite 即"一次评测的配方"**，由 TOML 文件描述（[asr-drama-wenetspeech.toml](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/suites/asr-drama-wenetspeech.toml)）：包含数据集、场景、参与对比的 arms（模型组合）。 +- **CLI 入口**： + ```bash + translip-lab run --suite asr-drama-wenetspeech --limit 32 + translip-lab compare --baseline --candidate + translip-lab doctor # 环境/数据集检查 + ``` +- **前端触发**：[ExperimentsTab](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L273-L357) 显示套件卡片，每张卡有： + - 套件名、数据集 + - 场景 chip + - **「试跑 (limit=3)」**按钮 → POST `/api/lab/runs` → 返回拼装好的 cmd +- **Run 状态**：4 种（`finished` / `running` / `failed` / `queued`），UI 用复用的 [StatusBadge](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/components/shared/StatusBadge.tsx) 呈现，running 行加左侧蓝条。 +- **Run 历史持久化**：[core/run_store.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/core/run_store.py)，文件系统型。 +- **缓存层**：[core/cache.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/core/cache.py) — 同输入相同 arm 命中缓存，加 `--no-cache` 绕过。 + +#### 4.3.2 需求来源 & 已知缺口 + +| 需求来源 | 是否覆盖 | 缺口 | +|---|---|---| +| 一键触发评测 | ✅ | 但目前是触发拼装命令，不是后端真跑（webhook 模式） | +| 查看历史 run | ✅ | — | +| Run 进度可视化 | ⚠️ | 只有 finished/running 二分，没有 % 进度 | +| 失败原因可追溯 | ❌ | 需要 UI 展示 stderr | +| 任务队列（多 run 排队） | ❌ | 目前是同步触发 | + +--- + +### 4.4 模块四：排行榜（Leaderboard） + +#### 4.4.1 已实现能力 + +- 自动从全部 run 中按**主指标** + **LOWER_IS_BETTER 集合**排序。 +- 🏆 标记当前最佳，其它行显示排名数字。 +- 展示列：`#` / `Run ID` / `Suite` / `模型 arm` / `场景 chip` / `状态` / `主指标值 + 指标名` / `RTF` / `创建时间` / `耗时`。 +- **响应式**：次要列在窄屏隐藏（`hidden md:table-cell` / `hidden lg:table-cell`），对齐 [TaskListPage](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/TaskListPage.tsx) 风格。 + +#### 4.4.2 需求来源 & 已知缺口 + +| 需求来源 | 是否覆盖 | 缺口 | +|---|---|---| +| 一眼看出"现在的最佳模型" | ✅ | — | +| 按场景过滤 | ❌ | 当前混排所有场景 | +| 按时间窗口过滤 | ❌ | — | +| 多指标可切换 | ❌ | 只看主指标 | +| 行点击进 run 详情页 | ❌ | 当前只展示，没有详情路由 | + +--- + +### 4.5 模块五：回归对比（Regression） + +#### 4.5.1 已实现能力 + +- **两个 select**：baseline & candidate，默认自动预选最近两个 `finished` run（[useMemo + finishedRuns[0/1]](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L499-L506)）。 +- **对比按钮触发**：调用 [labApi.compare](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/lab.ts#L173-L180)（mock 时走 [buildMockCompare](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/labMock.ts) 计算）。 +- **结果表格**展示 per_scenario： + - 场景 chip + 主指标名（+ "越大/越小越好"） + - baseline 值 / candidate 值 + - delta（绿色 ↓ improved / 红色 ↑ regressed / 灰色 - no change，由 [DeltaCell](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx#L464-L483) 可视化） + - 相对变化 % +- **Winner 判定**：自动算出谁更好或持平。 + +#### 4.5.2 需求来源 & 已知缺口 + +| 需求来源 | 是否覆盖 | 缺口 | +|---|---|---| +| 自动判定 winner | ✅ | — | +| 每个场景每个指标分别看 | ⚠️ | 当前只看每场景的"主指标"，不显示次要指标 | +| 多于 2 个 run 对比 | ❌ | 只能两两 | +| 导出对比报告 | ❌ | [report/markdown.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/report/markdown.py) 已有但前端没暴露 | + +--- + +### 4.6 模块六：体验保障层（前端基础设施） + +#### 4.6.1 已实现能力 + +- **Mock fallback**：[callOrFallback](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/lab.ts#L57-L79) — 后端不在/网络错误自动降级到 mock，非网络错误（4xx/5xx）继续抛。 +- **SourceBadge**：右上角实时显示「实时数据 / 演示数据」徽标，让用户始终知道当前看到的是不是真实运行结果。 +- **i18n 双语**：中英文完全对称的 ~30 个 lab 文案，集中在 [messages.ts](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/i18n/messages.ts#L2110-L2218)。 +- **URL 状态**：当前 Tab 写入 `?tab=xxx`，刷新保留。 +- **UI 规范对齐**：完全对齐项目 UI 规范（主蓝 `#3b5bdb`、卡片 `border-[#e5e7eb] shadow-[0_1px_3px_rgba(0,0,0,.04)]`、表头 `text-[#9ca3af] uppercase tracking-wide`、复用项目共享的 [PageContainer](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/components/layout/PageContainer.tsx) 与 [StatusBadge](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/components/shared/StatusBadge.tsx)）。 + +#### 4.6.2 关键设计原则 + +- 复用 > 重写：所有项目已有的共享原语（PageContainer、StatusBadge）优先复用，不再造轮子。 +- 视觉一致性 > 个性化：lab 是项目的一部分，不应跟主产品割裂。 +- 离线优先：mock fallback 不是"开发模式"，是产品演示能力。 + +--- + +## 5. 数据/接口契约 + +### 5.1 HTTP API（FastAPI in [server/app.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/server/app.py)） + +| 方法 | 路径 | 用途 | 前端调用方 | +|---|---|---|---| +| GET | `/api/lab/scenarios` | 列出所有场景定义 | （暂未在 UI 中使用） | +| GET | `/api/lab/datasets` | 列出已注册数据集 + 本地就绪状态 | DatasetsTab | +| GET | `/api/lab/suites` | 列出所有 suite 名 | ExperimentsTab | +| GET | `/api/lab/runs` | 列出所有 run | Leaderboard / Regression | +| GET | `/api/lab/runs/{run_id}` | 单个 run 详情（含 manifest、前 N 条样本） | （暂未在 UI 中使用） | +| GET | `/api/lab/compare?baseline=&candidate=` | 两个 run 的 per-scenario delta | RegressionTab | +| POST | `/api/lab/runs` | 触发一次 run（返回拼装好的 cmd） | ExperimentsTab | + +### 5.2 关键 TypeScript 接口（见 [lab.ts](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/lab.ts#L81-L142)） + +- `LabScenario` / `LabDataset` / `LabRunSummary` / `LabRunDetail` / `LabCompareResult` / `LabTriggerRunPayload` + +--- + +## 6. 已知问题与下一阶段路线图 + +### 6.1 P0（基础能力补齐） + +| 编号 | 需求 | 价值 | +|---|---|---| +| P0-1 | **Run 详情页**：点击 leaderboard 行进入，展示 manifest、前 N 条样本、stderr、所有指标 | 当前只能看汇总值，无法定位异常样本 | +| P0-2 | **失败可追溯**：failed run 的 stderr/log 在 UI 可见 | 调试效率 | +| P0-3 | **Lab server 真后端化**：当前触发是"拼装命令"，需让 server 真起 worker | 用户期望"点了就跑" | +| P0-4 | **Markdown 报告下载**：[report/markdown.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/report/markdown.py) 已有，前端 RegressionTab 接一个下载按钮即可 | 产出可分享物 | + +### 6.2 P1（产品力提升） + +| 编号 | 需求 | 价值 | +|---|---|---| +| P1-1 | **Leaderboard 按场景过滤** + 按时间窗口过滤 | 当前混排不利于做单场景决策 | +| P1-2 | **多 run 横向对比**（>2 个） | 选模型时一次看 4-5 个候选 | +| P1-3 | **数据集一键下载向导**（含 WenetSpeech EULA 引导） | 降低使用门槛 | +| P1-4 | **样本级 hypothesis vs reference diff** | 定位 CER 高来自哪些 sample | +| P1-5 | **指标选择器**：除主指标外，可看次要指标（CER + WER + RTF 同屏） | 全维度评估 | + +### 6.3 P2（生态化） + +| 编号 | 需求 | 价值 | +|---|---|---| +| P2-1 | **CI 钩子**：PR merge 自动跑 smoke suite，把结果回写到 PR 评论 | 真正闭环 | +| P2-2 | **自定义 suite 的可视化编辑器** | 不再写 TOML | +| P2-3 | **多用户 + 团队历史 run 隔离** | 公司内多团队共用一套部署 | +| P2-4 | **Run 标签 / 笔记**：给每个 run 加 label 与 markdown 备注 | 实验上下文留痕 | + +--- + +## 7. 非功能性需求 + +| 维度 | 要求 | 当前状态 | +|---|---|---| +| **离线可演示** | 无后端可看完整 UI | ✅ | +| **响应式** | 移动到 >=mobile 都能用 | ✅ 次要列 hidden 处理 | +| **i18n** | zh / en 双语对称 | ✅ | +| **可达性** | 颜色不是唯一信息载体（图标 + 文字） | ✅ ↑/↓ + 文字 | +| **可观测性** | 操作可追溯到具体 run_id | ✅ 行内显示 run_id | + +--- + +## 8. 风险与依赖 + +| 风险 | 影响 | 缓解 | +|---|---|---| +| WenetSpeech 下载体量大、需 EULA | 新成员上手慢 | mock 兜底已做；待补下载向导 | +| Lab server 当前只拼装命令 | 用户预期"点了就跑"会失望 | P0-3：真起 worker；或在 UI 明示"复制命令到终端运行" | +| 排行榜混排所有场景 | 决策时易误读 | P1-1 加场景过滤 | +| 缓存命中误判 | 改了模型但缓存没失效 | `--no-cache` 已有；前端需暴露开关 | + +--- + +## 9. 附录 + +### 9.1 当前模块代码地图 + +| 类别 | 路径 | +|---|---| +| **前端入口** | [LabPage.tsx](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/pages/lab/LabPage.tsx) | +| **前端 API + Mock** | [lab.ts](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/lab.ts) / [labMock.ts](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/api/labMock.ts) | +| **前端 i18n** | [messages.ts#L2110-L2218 (zh)](file:///Users/yinyijun/OpenSourceProjects/translip/frontend/src/i18n/messages.ts#L2110-L2218) | +| **后端包根** | [translip_lab](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab) | +| **CLI** | [cli.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/cli.py) | +| **HTTP Server** | [server/app.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/server/app.py) | +| **核心 Runner** | [core/runner.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/core/runner.py) | +| **场景** | [scenarios](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/scenarios) | +| **数据集** | [datasets](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/datasets) | +| **指标** | [metrics](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/metrics) | +| **报告** | [report](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/report) | +| **Suites 配置** | [suites](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/suites) | +| **测试** | [tests/lab](file:///Users/yinyijun/OpenSourceProjects/translip/tests/lab) | + +### 9.2 启动命令 + +```bash +# 1) 仅看前端产品形态（mock 数据） +cd frontend +VITE_LAB_USE_MOCK=1 npm run dev +# → 浏览器打开 http://localhost:5173/lab + +# 2) 起后端 + 真实数据 +translip-lab-server # 默认 :8799 +cd frontend && npm run dev # 自动连接 + +# 3) CLI 直跑 +translip-lab doctor +translip-lab run --suite asr-drama-wenetspeech --limit 32 +translip-lab compare --baseline --candidate +``` + +### 9.3 文档版本约定 + +- 本文档反映 **2026-06-22** 时刻代码状态。 +- 之后每个 P0/P1/P2 项落地后，在第 6 节标注 ✅，并把新增能力补到第 4 节对应小节。 + From cfec30e1dcde537b71529bc81ce656c02f52fb44 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 10:47:28 +0800 Subject: [PATCH 04/10] feat(lab): add tts-clone (SIM+CER) and MagicData-RAMC eval coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new GT-anchored evaluation paths, both offline-tested with fabricated data (no downloads), filling the two biggest gaps from the dataset-landscape research. tts-clone — the dubbing/TTS stage had no GT metric, only e2e-dub's intrinsic score: - metrics/speaker.py: SIM = cosine over ECAPA embeddings (reuses translip's in-tree speaker_embedding; lazy + injectable + degrades to sim=None, never crashes) - scenarios/tts_clone.py: clone reference voice -> re-transcribe -> SIM (primary) + CER - tts_synth.py: lab-side worker reusing translip.dubbing qwen voice-clone core only (avoids the server adapters' heavy eager-import), isolated subprocess - datasets/synthetic_clone.py + folder .clone.txt/.ref.wav sidecars + suite - GroundTruth gains clone_text / clone_ref_wav magicdata-ramc — extends CER+DER from meetings into spontaneous dialogue: - datasets/magicdata_ramc.py: dedicated parser for RAMC's "[start,end] speaker gender,dialect text" transcripts (NOT TextGrid), reusing the shared RTTM/SRT emitters; tolerant of flat or WAV/TXT layouts - suites/asr-diar-ramc.toml (collar 0.25 / 2 speakers -> official CER 19.1/DER 7.96) Tests: +19 (tests/lab now 104 passing). Design doc at docs/translip-lab-tts-clone-eval.zh-CN.md. Format + real-TTS paths flagged as "verify against real data/models" in describe()/docs (cannot download here). translip core untouched - lab stays one-way-dependent and deletable. --- docs/translip-lab-tts-clone-eval.zh-CN.md | 192 ++++++++++++++++++ src/translip_lab/README.md | 23 ++- src/translip_lab/cli.py | 2 +- src/translip_lab/core/sample.py | 4 + src/translip_lab/datasets/__init__.py | 2 +- src/translip_lab/datasets/folder.py | 10 +- src/translip_lab/datasets/magicdata_ramc.py | 141 +++++++++++++ src/translip_lab/datasets/synthetic_clone.py | 85 ++++++++ src/translip_lab/metrics/__init__.py | 2 + src/translip_lab/metrics/speaker.py | 85 ++++++++ src/translip_lab/scenarios/__init__.py | 2 +- src/translip_lab/scenarios/tts_clone.py | 105 ++++++++++ src/translip_lab/suites/asr-diar-ramc.toml | 29 +++ .../suites/tts-clone-synthetic.toml | 18 ++ src/translip_lab/tts_synth.py | 87 ++++++++ tests/lab/test_datasets_ramc.py | 99 +++++++++ tests/lab/test_datasets_synthetic_clone.py | 43 ++++ tests/lab/test_metrics_speaker.py | 48 +++++ tests/lab/test_scenario_tts_clone.py | 78 +++++++ 19 files changed, 1045 insertions(+), 10 deletions(-) create mode 100644 docs/translip-lab-tts-clone-eval.zh-CN.md create mode 100644 src/translip_lab/datasets/magicdata_ramc.py create mode 100644 src/translip_lab/datasets/synthetic_clone.py create mode 100644 src/translip_lab/metrics/speaker.py create mode 100644 src/translip_lab/scenarios/tts_clone.py create mode 100644 src/translip_lab/suites/asr-diar-ramc.toml create mode 100644 src/translip_lab/suites/tts-clone-synthetic.toml create mode 100644 src/translip_lab/tts_synth.py create mode 100644 tests/lab/test_datasets_ramc.py create mode 100644 tests/lab/test_datasets_synthetic_clone.py create mode 100644 tests/lab/test_metrics_speaker.py create mode 100644 tests/lab/test_scenario_tts_clone.py diff --git a/docs/translip-lab-tts-clone-eval.zh-CN.md b/docs/translip-lab-tts-clone-eval.zh-CN.md new file mode 100644 index 0000000..0f7cb93 --- /dev/null +++ b/docs/translip-lab-tts-clone-eval.zh-CN.md @@ -0,0 +1,192 @@ +# translip-lab：配音/音色克隆评测（tts-clone）设计文档 + +> 文档版本：v1.0 · 日期：2026-06-23 +> 范围：在 `translip_lab` 评测台中新增 **`tts-clone`** 场景 + **说话人相似度（SIM）/可懂度（CER）** 指标 + **`synthetic-clone`** 合成数据集，把「配音/TTS」环节从「无客观 ground-truth 打分」补成「可量化、可回归、可对比」。 +> 关联调研：`中文影视剧/邻近语料数据集调研`（seed-tts-eval 的 WER+SIM 方法学）；关联现状：Lab 模块 PRD（`docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md`）。 + +--- + +## 1. 背景与要解决的问题 + +`translip_lab` 当前对 5 个环节有真值打分（分离 SI-SDR、ASR CER、说话人 DER、OCR 文本/框 F1、字幕擦除 PSNR/SSIM），但**配音/TTS 这条核心能力没有独立的客观指标**： + +- `e2e-dub` 场景（`scenarios/e2e_dub.py`）的主指标是 translip 自带的 **intrinsic「honest score」**（`benchmark-dub` 的 0–100 分），**不依赖任何外部 ground truth**。 +- 该 intrinsic 分有已知盲点：它是一个偏薄的 JOIN，会**漏报 0.25–0.45 的音色相似度带、把节奏问题混进总分**（见既往 dub-eval 复盘）。把评测台的配音头牌指标压在它上面，等于把它的盲点一起带进 leaderboard。 +- PRD（§4.2.1）把 `e2e_dub` 主指标列为 **`mcd`（越小越好）**，但代码里实际是 `primary_metric_key = "score"`（intrinsic，越大越好）——**说明作者本想要一个 GT 锚定的客观指标，但最终没落地**。 + +**本设计提供那个「本想要却没落地」的客观指标**：参照 2024+ 零样本音色克隆的事实标准 **seed-tts-eval**，用两个互补的、有 ground truth 的指标评配音： + +| 指标 | 含义 | 真值 | 复用 | +|---|---|---|---| +| **SIM**（说话人相似度） | 合成音是否保住了**目标音色** | 参考音色 wav | translip 自带 **ECAPA** 嵌入（`speaker_embedding.py`） | +| **CER**（可懂度） | 合成音**说对了没有**（重转写后比对目标文本） | 目标文本 | translip ASR（`transcribe`）+ lab 的 `metrics/text.py` | + +> 设计取向：**SIM 设为 `tts-clone` 的主指标**（越大越好）。理由——CER 已被 `asr` 场景覆盖，而 SIM（音色保真）恰是 intrinsic 分**最会漏报**、且当前**全台没有任何指标覆盖**的维度。CER 作为次要指标同屏报出。 + +--- + +## 2. 设计总览 + +完全遵循 Lab 既有架构「加能力＝加一个文件」，不改动核心引擎、不破坏单向依赖： + +``` +新增： + metrics/speaker.py cosine_similarity + speaker_similarity（懒加载 ECAPA，可注入 embedder） + datasets/synthetic_clone.py synthetic-clone：造「参考音色 wav + 目标文本」配对（离线可测） + scenarios/tts_clone.py tts-clone：invoke（TTS→转写）+ score（SIM + CER） + tts_synth.py lab 侧 TTS worker（python -m），镜像 translip 语音克隆核心 + suites/tts-clone-synthetic.toml + tests/lab/test_metrics_speaker.py + tests/lab/test_datasets_synthetic_clone.py + tests/lab/test_scenario_tts_clone.py + +改动（最小）： + core/sample.py GroundTruth 增 clone_text / clone_ref_wav 两字段 + metrics/__init__.py 导出 cosine_similarity / speaker_similarity + datasets/__init__.py import synthetic_clone（注册） + scenarios/__init__.py import tts_clone（注册） + datasets/folder.py 识别 .clone.txt / .ref.wav 边车（自带数据走真实路径） + cli.py gen-synthetic 增 --kind clone + README.md 场景表补 tts-clone 行 +``` + +--- + +## 3. 数据契约 + +### 3.1 GroundTruth 新增字段（`core/sample.py`） + +```python +clone_text: str | None = None # 要合成的目标文本 → CER 真值 +clone_ref_wav: Path | None = None # 目标音色参考 wav → SIM 真值（缺省回退到 sample.media_path） +``` + +两者皆 optional，与既有字段一致：场景按需校验（`tts-clone` 的 `required_gt = ["clone_text"]`）。 + +### 3.2 `synthetic-clone` 合成数据布局（离线可造、可测） + +每个 case 在 `/synthetic-clone/clip_NNN/` 生成： + +``` +prompt.wav 一段「带说话人特征（共振峰由 speaker_seed 决定）」的合成参考音色 + —— 既是 sample.media_path（scenario 的输入参考音），也是 SIM 真值 +meta: clone_text = 取自固定中文句库（按 index 轮换，确定性） +``` + +- 同一 `speaker_seed` → 同一音色（共振峰一致）；不同 seed → 不同音色。供测试构造「高 SIM / 低 SIM」对照。 +- 纯 numpy + soundfile 合成，**不需要任何模型、不需要下载**，与既有 `synthetic-mix` / `synthetic-subtitle` 同一思路（验证管路与指标；真实数字用 `folder` 喂真实素材）。 + +### 3.3 `folder` 真实数据边车（自带影视素材走真实路径） + +为 `.` 增加两个可选边车： + +``` +.clone.txt 目标文本（要让 TTS 用该片段音色去说的话） +.ref.wav 目标音色参考（可选；缺省用媒体自身做参考） +``` + +这样把「自建中文影视音色样本」直接变成可量化 case，零摩擦（呼应调研里「自建影视评测集」的落地建议）。 + +--- + +## 4. 场景执行流（`scenarios/tts_clone.py`） + +``` +invoke(sample, work_dir, invoker, config): + ref = sample.ground_truth.clone_ref_wav or sample.media_path + text = sample.ground_truth.clone_text + # 1) TTS 语音克隆（隔离子进程，复用 translip 核心） + r_tts = invoker.module("translip_lab.tts_synth", + ["--text", text, "--reference", ref, "--output", work_dir/"synth.wav", + "--backend", config.tts_backend or "qwen3tts", "--language", lang]) + if not r_tts.ok: return r_tts # 失败 → 场景标 failed（不崩整轮） + synth = r_tts.outputs["synth_wav"] + # 2) 重转写合成音（可懂度用），复用 translip ASR + r_asr = invoker.translip("transcribe", ["--input", synth, "--language", lang, ...]) + r_asr.outputs["synth_wav"] = synth # 把合成音路径并入返回的 outputs + return r_asr + +score(sample, work_dir, stage, config): + synth = stage.outputs["synth_wav"]; segs = stage.outputs["segments"] + hyp_text = "".join(seg.text for seg in load(segs)) + cer = metrics.text.cer(sample.ground_truth.clone_text, hyp_text) + sim = metrics.speaker.speaker_similarity(ref, synth)["sim"] # ECAPA 余弦；不可用→None + return {"sim": sim, "cer": cer, "wer": wer, "intelligibility": 1-min(cer,1), ...} +``` + +- **主指标**：`sim`（`higher_is_better = True`）。`cer` 次要同屏。 +- **优雅降级**：若 ECAPA 不可用（理论上不会，它是 diarization 默认后端的同款依赖），`sim=None` 并记一条 note，场景仍成功返回 `cer`——不阻断。 +- **corpus 聚合**（`corpus_metrics`）：`cer_micro`（池化）+ `sim_mean`。 + +--- + +## 5. lab 侧 TTS worker（`tts_synth.py`，`python -m translip_lab.tts_synth`） + +为什么是 lab 侧 worker 而非直接复用 `server/atomic_tools` 的 `generate_speech`：后者所在的 `adapters/__init__.py` 会 **eager-import 全部重适配器**（separation/vision…，拖入 demucs/torch/vision），违背 lab「base 依赖 + 轻量」原则。 + +worker **只导入 `translip.dubbing` core**，镜像原子工具的 `_generate_voice_clone`（qwen3tts 路径）： + +```python +from translip.dubbing.backend import SynthSegmentInput, resolve_tts_device +from translip.dubbing.qwen_tts_backend import ( + _load_qwen_model, _language_name, _max_new_tokens_for, _normalize_waveform) +# device → 加载 Qwen3-TTS-Base → generate_voice_clone(text, ref_audio=...) → 归一化 → 写 wav +# stdout 机器可读：synth_wav= sample_rate= tts_backend=qwen3tts +``` + +- **单向依赖**：lab → translip.dubbing，保持「translip 不反向依赖 lab」。 +- **子进程隔离**：多 GB 的 TTS 模型用完即随子进程退出释放（与 orchestrator/lab 既有姿势一致）。 +- **后端范围**：v1 落 **qwen3tts**（默认、最干净）。`moss-tts-nano-onnx` / `voxcpm2` 需走 `ReferencePackage` 参考音预处理，列为后续（需真实模型 smoke test，见 §9）。 + +--- + +## 6. 复用与解耦（守住 Lab 的「一条规则」） + +| 复用点 | 来源 | 性质 | +|---|---|---| +| 说话人嵌入（SIM） | `translip.speaker_embedding`（ECAPA/speechbrain） | 纯函数导入，与既有「借用 `translip.transcription.benchmark`」同姿势 | +| 语音克隆 TTS | `translip.dubbing.qwen_tts_backend`（core） | 子进程 `python -m`，模型用完释放 | +| 可懂度转写 | `translip transcribe`（CLI） | 子进程，与其它场景一致 | +| CER/cosine | lab `metrics/text.py`、`metrics/speaker.py` | 纯 numpy/stdlib，可单测 | + +依旧满足：**删掉 `src/translip_lab/` + 一个 sidebar link，主系统无感**。 + +--- + +## 7. 离线测试方案（用合成数据，不下载、不跑真模型） + +三层，全部离线、确定性： + +1. **指标层** `test_metrics_speaker.py` + - `cosine_similarity`：相同向量→1，正交→0，反向→−1。 + - `speaker_similarity` 用**注入的纯 numpy embedder**：相同音→sim≈1，不同音→更低；embedder 抛错→`sim=None` 且不崩（优雅降级）。 +2. **数据层** `test_datasets_synthetic_clone.py` + - 生成器产出 `prompt.wav` + manifest 带 `clone_text` + `clone_ref_wav`；确定性（同 seed 同输出）；同 speaker_seed 的两段音 MFCC 距离 < 不同 seed。 +3. **场景层** `test_scenario_tts_clone.py`（核心，端到端打分路径全覆盖） + - 用**假 Invoker**（`StageResult` 注入 `synth_wav` + 转写 `segments`）+ **monkeypatch 掉默认 ECAPA embedder 为纯 numpy**： + - 完美档：合成音=参考音、转写=目标文本 → `cer≈0`、`sim` 高。 + - 退化档：转写乱码 → `cer` 高；合成音=不同说话人 → `sim` 低。 + - 断言 `tts-clone` 已注册、`required_gt` 校验、缺 `clone_text` 时 `skipped`。 + +> 真实 TTS 的 `invoke` 路径（需 Qwen3-TTS 模型）**不在离线测试范围**，作为 §9 的 smoke-test 项；但 `score`（真正补的能力）100% 离线可测。 + +--- + +## 8. 与既有 `e2e-dub` 的关系 + +- **不改 `e2e_dub.py`**（保持 intrinsic 回归分作为端到端冒烟）。 +- `tts-clone` 是**环节级、GT 锚定**的客观指标，补 intrinsic 分漏报的音色维度——即 PRD 里 `mcd` 想做却没做的那个客观指标的更好实现（SIM+CER ＞ 单一 intrinsic 分）。 +- 两者并存：`e2e-dub` 看「整条链能不能跑、整体回归」，`tts-clone` 看「TTS 这一环音色保真 + 可懂度」。 + +--- + +## 9. 落地清单与后续 + +**本次落地（离线可测）**：metrics/speaker.py · datasets/synthetic_clone.py · scenarios/tts_clone.py · tts_synth.py · suite · GroundTruth 字段 · folder 边车 · CLI · 三个测试文件。 + +**后续（需数据/模型，硬盘就绪后）**： +1. **moss/voxcpm2** 后端的 worker 路径 + 真实模型 smoke test。 +2. **MagicData-RAMC** 适配器 — **✅ 已落地**：`datasets/magicdata_ramc.py`（专用 `[start,end] speaker 性别,方言 text` 解析器，复用 RTTM/SRT 发射器）+ `suites/asr-diar-ramc.toml` + `tests/lab/test_datasets_ramc.py`（6 个 fixture 测试验证解析逻辑）。CER+DER 扩到自发对话域。⚠️ 解析格式据官方发布示例构建，**真数据首跑前抽一条 `.txt` 逐字节对照**（解析器已隔离便于微调）。 +3. **ChaLearn Decaptioning** 适配器（subtitle-erase 跨集校验；调研 P1）。 +4. **缓存键纳入 scorer 版本**（`cache.py` 现仅按 config+input 哈希，改打分逻辑不失效——既有 footgun；加 `Scenario.version` 进 key）。 +5. 自建中文影视音色小集（走 §3.3 的 `folder` 边车）。 diff --git a/src/translip_lab/README.md b/src/translip_lab/README.md index 7c46105..84d77f9 100644 --- a/src/translip_lab/README.md +++ b/src/translip_lab/README.md @@ -2,14 +2,15 @@ A **loosely-coupled** harness that tests/optimizes existing `translip` capabilities against external, ground-truth-annotated datasets and produces **quantitative, -comparable** results (CER / DER / SI-SDR / PSNR-SSIM / detection-F1). +comparable** results (CER / DER / SI-SDR / PSNR-SSIM / detection-F1 / speaker-SIM). ## Loose coupling (the one rule) `translip_lab` depends on `translip` **one way only** — translip never imports the lab. The integration surface is the stable translip **CLI/JSON contract** (run via subprocess, exactly like the orchestrator) plus a couple of pure helper imports -(`translip.transcription.benchmark` for ASR scoring). Delete `src/translip_lab/` +(`translip.transcription.benchmark` for ASR scoring, `translip.speaker_embedding` +for the tts-clone SIM metric). Delete `src/translip_lab/` and the single "Testing Lab" link in the frontend sidebar, and the main system is untouched. The dashboard runs as its **own service on its own port** — the main UI just links to it. @@ -116,16 +117,25 @@ Reproducible via the suites above, on real corpora under `/Volumes/EXT`: | `ocr-detect` | `python -m translip.ocr.extract` | box F1 ↑ | subtitle box GT | | `subtitle-erase` | `… ocr.extract` → `erase.extract` | PSNR/SSIM ↑ | clean (sub-free) video | | `e2e-dub` | `run-pipeline` → `benchmark-dub` | honest score ↑ | (intrinsic, no GT) | +| `tts-clone` | `python -m translip_lab.tts_synth` → `transcribe` | SIM ↑ (+ CER ↓) | reference voice + target text | ## Datasets - **`folder`** — bring your own: drop media + sidecars under a dir. Per ``: `.srt` (ASR), `.rttm` (diar), `.boxes.json` (OCR), `.clean.mp4` (erase), - `.voice.wav`+`.background.wav` (separation). + `.voice.wav`+`.background.wav` (separation), `.clone.txt` + optional `.ref.wav` + (tts-clone: target text + reference voice). - **`aishell4`** (SLR111) / **`alimeeting`** (SLR119) — Mandarin meetings, ASR+diar GT. Place under `/aishell4//{wav,TextGrid}` and `/alimeeting//{audio_dir,textgrid_dir}`. **Domain caveat:** meeting/telephone, the only rigorous open CER/DER GT — not film/TV. +- **`magicdata-ramc`** (SLR123) — Mandarin **spontaneous 2-party conversation**, + ASR + diar GT in one corpus (manual transcript + per-speaker timestamps). Extends + CER/DER from meetings into the *dialogue* domain (closer to film/TV than meetings). + Extract under `/magicdata-ramc//`; the adapter parses RAMC's + `[start,end] speaker gender,dialect text` transcripts (**not** TextGrid). License + CC BY-NC-ND 4.0 (academic). Baseline CER 19.1% / DER 7.96% (collar 0.25). Run via + `translip-lab run --suite asr-diar-ramc`. - **`wenetspeech-drama`** — the "D" (drama) subset of WenetSpeech, Mandarin film/TV ASR with reference transcripts. EULA-gated: register at https://wenet.org.cn/WenetSpeech/, filter `WenetSpeech.json` by @@ -134,9 +144,10 @@ Reproducible via the suites above, on real corpora under `/Volumes/EXT`: (schema documented at the top of `datasets/wenetspeech_drama.py`). This is the first dataset that puts the film/TV domain on the leaderboard — run it via `translip-lab run --suite asr-drama-wenetspeech`. -- **`synthetic-subtitle`** / **`synthetic-mix`** — generated GT for OCR/erase and - separation (no public CN film/TV GT exists for these). Synthetic numbers validate - plumbing; for real numbers supply real material via `folder`. +- **`synthetic-subtitle`** / **`synthetic-mix`** / **`synthetic-clone`** — generated GT + for OCR/erase, separation, and voice-clone TTS (no public CN film/TV GT exists for + these). Synthetic numbers validate plumbing; for real numbers supply real material + via `folder`. ## Design diff --git a/src/translip_lab/cli.py b/src/translip_lab/cli.py index 3c8df3e..91ff02f 100644 --- a/src/translip_lab/cli.py +++ b/src/translip_lab/cli.py @@ -277,7 +277,7 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("scenarios", help="list capability scenarios") g = sub.add_parser("gen-synthetic", help="generate synthetic GT datasets into the cache") - g.add_argument("--kind", choices=["subtitle", "mix", "both"], default="both") + g.add_argument("--kind", choices=["subtitle", "mix", "clone", "both"], default="both") g.add_argument("--clips", type=int, default=1) g.add_argument("--duration", type=float, default=4.0) diff --git a/src/translip_lab/core/sample.py b/src/translip_lab/core/sample.py index 4fa7ec4..b3a5b1d 100644 --- a/src/translip_lab/core/sample.py +++ b/src/translip_lab/core/sample.py @@ -28,6 +28,8 @@ class GroundTruth: subtitle_boxes: Path | None = None # JSON {events:[{start,end,text,box}]} → OCR detection F1 clean_video: Path | None = None # subtitle-free reference video → erase PSNR/SSIM clean_frames_dir: Path | None = None # or pre-extracted clean frames + clone_text: str | None = None # target text to synthesize → tts-clone CER (intelligibility) + clone_ref_wav: Path | None = None # target-voice reference → tts-clone SIM (timbre); falls back to media extra: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: @@ -38,6 +40,8 @@ def to_dict(self) -> dict[str, Any]: "subtitle_boxes": _opt_str(self.subtitle_boxes), "clean_video": _opt_str(self.clean_video), "clean_frames_dir": _opt_str(self.clean_frames_dir), + "clone_text": self.clone_text, + "clone_ref_wav": _opt_str(self.clone_ref_wav), "extra": dict(self.extra), } diff --git a/src/translip_lab/datasets/__init__.py b/src/translip_lab/datasets/__init__.py index d3ccb83..d28a9cf 100644 --- a/src/translip_lab/datasets/__init__.py +++ b/src/translip_lab/datasets/__init__.py @@ -4,6 +4,6 @@ from .base import DATASET_REGISTRY, DatasetAdapter, available_datasets, get_dataset, register_dataset # Importing each module registers its adapter. -from . import aishell4, alimeeting, clip, folder, synthetic_mix, synthetic_subtitle, textgrid_folder, wenetspeech_drama # noqa: E402,F401 +from . import aishell4, alimeeting, clip, folder, magicdata_ramc, synthetic_clone, synthetic_mix, synthetic_subtitle, textgrid_folder, wenetspeech_drama # noqa: E402,F401 __all__ = ["DATASET_REGISTRY", "DatasetAdapter", "get_dataset", "register_dataset", "available_datasets"] diff --git a/src/translip_lab/datasets/folder.py b/src/translip_lab/datasets/folder.py index 348b331..c468763 100644 --- a/src/translip_lab/datasets/folder.py +++ b/src/translip_lab/datasets/folder.py @@ -6,6 +6,9 @@ .boxes.json → subtitle box GT (OCR detection F1) .clean.mp4 → subtitle-free reference video (erase PSNR/SSIM) .voice.wav + .background.wav → clean stems (separation SI-SDR) + .clone.txt → target text for voice-clone TTS (tts-clone CER); the media + itself is the reference voice unless .ref.wav is present + .ref.wav → explicit target-voice reference (tts-clone SIM) This is how you turn ANY corpus (or your own clips) into a quantifiable bench once you drop the references next to the media. @@ -20,7 +23,7 @@ from .base import DatasetAdapter, register_dataset _MEDIA_EXTS = (".mp4", ".mkv", ".mov", ".webm", ".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg") -_GT_MEDIA_SUFFIXES = (".voice", ".background", ".clean") +_GT_MEDIA_SUFFIXES = (".voice", ".background", ".clean", ".ref") @register_dataset @@ -68,6 +71,11 @@ def normalize(self) -> SampleManifest: background = Path(f"{stem_path}.background.wav") if voice.is_file() and background.is_file(): gt.clean_stems = {"voice": str(voice), "background": str(background)} + clone_txt = Path(f"{stem_path}.clone.txt") + if clone_txt.is_file(): + gt.clone_text = clone_txt.read_text(encoding="utf-8").strip() + ref_wav = Path(f"{stem_path}.ref.wav") + gt.clone_ref_wav = ref_wav if ref_wav.is_file() else media samples.append(Sample( sample_id=media.stem, media_path=media, ground_truth=gt, meta={"lang": self.lang, "source": "folder"}, diff --git a/src/translip_lab/datasets/magicdata_ramc.py b/src/translip_lab/datasets/magicdata_ramc.py new file mode 100644 index 0000000..e050205 --- /dev/null +++ b/src/translip_lab/datasets/magicdata_ramc.py @@ -0,0 +1,141 @@ +"""MagicData-RAMC adapter (OpenSLR SLR123) — Mandarin spontaneous *conversation*, +ASR (CER) + diarization (DER) GT in one corpus. + +Why this corpus: the lab's open CER/DER GT was meetings only (AISHELL-4 / AliMeeting) +plus film/TV ASR (WenetSpeech-Drama, no speaker GT). RAMC is 180h of two-party phone +conversations with manual transcripts AND per-speaker voice-activity timestamps, so +it extends both CER and DER into the spontaneous-dialogue domain — far closer to +film/TV dialogue dynamics than meetings. Official baselines: CER 19.1% (test), +DER 7.96% (collar 0.25). License: CC BY-NC-ND 4.0 (academic/eval, not commercial). + +**Format — NOT TextGrid.** RAMC ships per-conversation ``.txt`` transcripts whose +segments are:: + + [start,end] + e.g. [1.319,6.691] G00000140 女,普通话 爱数智慧语音采集二零一九年十一月六日 + +plus ``SPKINFO.txt`` / ``UTTERANCEINFO.txt`` metadata. This adapter parses those +segments into ``(start, end, speaker, text)`` and reuses the shared RTTM/SRT +emitters. Built to the published format examples; **confirm against one real +conversation file before trusting real-data numbers** — the parser is isolated +here (``parse_ramc_transcript``) for easy adjustment. + +Place the extracted corpus under ``/magicdata-ramc//`` (subset = +``train`` | ``dev`` | ``test``). The adapter finds every ``*.wav`` and its sibling +``.txt`` anywhere under that dir, so it tolerates flat or WAV/TXT-split layouts. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from ..config import LabConfig +from ..core.sample import GroundTruth, Sample, SampleManifest +from .base import DatasetAdapter, register_dataset +from .textgrid import to_rttm, to_srt + +_SEGMENT = re.compile(r"\[\s*([0-9]+(?:\.[0-9]+)?)\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\]") +# A gender,dialect attribute token like 女,普通话 / 男,四川话 — distinct from transcription text. +_ATTRS = re.compile(r"^[男女][^\s,]*,[^\s]+$") +_METADATA_NAMES = {"SPKINFO.txt", "UTTERANCEINFO.txt"} + + +def parse_ramc_transcript(path: str | Path) -> list[tuple[float, float, str, str]]: + """Parse a RAMC ``.txt`` into ``[(start, end, speaker, text), ...]``, sorted by time. + + Splits on each ``[start,end]`` marker and reads the trailing + `` [] ``. The optional demographic token is + dropped from the reference text; whitespace inside the (Chinese) text is + collapsed — CER ignores whitespace anyway. Zero-length / empty segments drop. + """ + content = Path(path).read_text(encoding="utf-8", errors="replace") + marks = list(_SEGMENT.finditer(content)) + intervals: list[tuple[float, float, str, str]] = [] + for i, m in enumerate(marks): + start, end = float(m.group(1)), float(m.group(2)) + chunk = content[m.end(): (marks[i + 1].start() if i + 1 < len(marks) else len(content))] + tokens = chunk.split() + if len(tokens) < 2: + continue # need at least a speaker + one text token + speaker = tokens[0] + rest = tokens[1:] + if rest and _ATTRS.match(rest[0]): + rest = rest[1:] # strip the gender,dialect attribute + text = "".join(rest).strip() + if end > start and text: + intervals.append((start, end, speaker, text)) + intervals.sort(key=lambda x: (x[0], x[1])) + return intervals + + +@register_dataset +class MagicDataRamcDataset(DatasetAdapter): + name = "magicdata-ramc" + + def __init__(self, config: LabConfig, *, subset: str = "test", audio_ext: str = ".wav", + lang: str = "zh", **params: Any) -> None: + super().__init__(config, subset=subset, audio_ext=audio_ext, lang=lang, **params) + self.subset = subset + self.audio_ext = audio_ext if audio_ext.startswith(".") else f".{audio_ext}" + self.lang = lang + self._base = config.datasets_dir / "magicdata-ramc" / subset + + @property + def root(self) -> Path: + return self.config.datasets_dir / "magicdata-ramc" + + def _cache_dir(self) -> Path: + d = self.config.cache_dir / "datasets" / self.name / self.subset + d.mkdir(parents=True, exist_ok=True) + return d + + def _index_transcripts(self) -> dict[str, Path]: + index: dict[str, Path] = {} + for txt in self._base.rglob("*.txt"): + if txt.name in _METADATA_NAMES: + continue + index.setdefault(txt.stem, txt) + return index + + def normalize(self) -> SampleManifest: + if not self._base.exists(): + raise FileNotFoundError( + f"MagicData-RAMC subset not found: {self._base}. Extract the OpenSLR " + "SLR123 archive there (see `translip-lab datasets` for the layout)." + ) + transcripts = self._index_transcripts() + cache = self._cache_dir() + samples: list[Sample] = [] + for audio in sorted(self._base.rglob(f"*{self.audio_ext}")): + gt = GroundTruth() + txt = transcripts.get(audio.stem) + if txt is not None: + intervals = parse_ramc_transcript(txt) + out_dir = cache / audio.stem + out_dir.mkdir(parents=True, exist_ok=True) + rttm_path = out_dir / "ref.rttm" + rttm_path.write_text(to_rttm(intervals, audio.stem), encoding="utf-8") + srt_path = out_dir / "ref.srt" + srt_path.write_text(to_srt(intervals), encoding="utf-8") + gt.rttm = rttm_path + gt.transcript_srt = srt_path + samples.append(Sample( + sample_id=audio.stem, media_path=audio, ground_truth=gt, + meta={"lang": self.lang, "source": self.name, "subset": self.subset}, + )) + return SampleManifest(dataset=self.name, samples=samples, + meta={"base": str(self._base), "subset": self.subset, "lang": self.lang}) + + def describe(self) -> dict[str, Any]: + d = super().describe() + d.update({ + "license": "CC BY-NC-ND 4.0 (OpenSLR SLR123; academic/eval, not commercial)", + "provides": ["asr (CER)", "diarization (DER)"], + "domain": "spontaneous 2-party phone conversation (16 kHz)", + "expected_layout": f"{self.root}//**/*.wav + sibling .txt" + " ([start,end] speaker gender,dialect text)", + "baseline": "CER 19.1% / DER 7.96% (collar 0.25), test set", + "caveat": "transcript format built to published examples — confirm against a real .txt first", + }) + return d diff --git a/src/translip_lab/datasets/synthetic_clone.py b/src/translip_lab/datasets/synthetic_clone.py new file mode 100644 index 0000000..00a4705 --- /dev/null +++ b/src/translip_lab/datasets/synthetic_clone.py @@ -0,0 +1,85 @@ +"""Synthetic voice-clone cases — (reference voice, target text) pairs for tts-clone. + +No public Chinese film/TV clone-eval set exists, so we fabricate a deterministic +"speaker" (a formant-structured harmonic tone whose timbre is fixed by +``speaker_seed``) as the reference voice and pair it with a target sentence. This +validates the ``tts-clone`` plumbing + SIM/CER metrics offline (no model, no +download), exactly like ``synthetic-mix`` / ``synthetic-subtitle`` do for their +stages. For *real* numbers feed real voice samples via the ``folder`` dataset +(``.clone.txt`` + optional ``.ref.wav``). +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import numpy as np + +from ..config import LabConfig +from ..core.sample import GroundTruth, Sample, SampleManifest +from .base import DatasetAdapter, register_dataset + +# Short, varied Mandarin lines (dialogue-like) so CER reflects real-ish phonetics. +_SENTENCES = ( + "今天的天气非常好，我们一起去公园散步吧。", + "他缓缓转过身，眼神里写满了不舍。", + "这件事我会负责到底，请你放心。", + "列车即将到站，请乘客们提前做好准备。", + "她轻声说，谢谢你一直陪在我身边。", +) + + +def synth_voice(seed: int, *, duration: float = 4.0, sr: int = 16000) -> np.ndarray: + """A deterministic 'speaker': harmonic tone whose pitch + formant weights depend on seed. + + Same seed → same timbre (high SIM); different seed → different timbre (lower + SIM). Pure numpy, so embeddings of two same-seed clips land close together. + """ + rng = np.random.default_rng(seed) + t = np.linspace(0, duration, int(sr * duration), endpoint=False) + f0 = 110.0 + (seed % 7) * 14.0 # speaker-dependent fundamental + weights = 0.6 + 0.4 * np.abs(rng.standard_normal(4)) # speaker-dependent harmonic timbre + syllable = 0.5 * (1.0 + np.sin(2 * np.pi * 3.3 * t)) # ~3.3 Hz syllable envelope + voice = np.zeros_like(t) + for k, base_amp in enumerate((0.6, 0.3, 0.18, 0.1), start=1): + voice += base_amp * weights[k - 1] * np.sin(2 * np.pi * f0 * k * t) + voice *= syllable + peak = float(np.max(np.abs(voice))) or 1.0 + return (voice / peak * 0.7).astype(np.float32) + + +def generate_clone_case(out_dir: Path, *, index: int = 0, speaker_seed: int = 0, + duration: float = 4.0, sr: int = 16000) -> dict[str, Any]: + """Write ``prompt.wav`` (the reference voice) and return its path + target text.""" + import soundfile as sf + + out_dir.mkdir(parents=True, exist_ok=True) + prompt = out_dir / "prompt.wav" + sf.write(prompt, synth_voice(speaker_seed, duration=duration, sr=sr), sr) + return {"prompt": prompt, "text": _SENTENCES[index % len(_SENTENCES)]} + + +@register_dataset +class SyntheticCloneDataset(DatasetAdapter): + name = "synthetic-clone" + + def __init__(self, config: LabConfig, *, clips: int = 2, duration: float = 4.0, + sr: int = 16000, **params: Any) -> None: + super().__init__(config, clips=clips, duration=duration, sr=sr, **params) + self.clips = clips + self.duration = duration + self.sr = sr + + def normalize(self) -> SampleManifest: + out_root = self.config.cache_dir / "synthetic-clone" + samples: list[Sample] = [] + for i in range(self.clips): + case = generate_clone_case(out_root / f"clip_{i:03d}", index=i, speaker_seed=i, + duration=self.duration, sr=self.sr) + gt = GroundTruth(clone_text=case["text"], clone_ref_wav=case["prompt"]) + samples.append(Sample( + sample_id=f"synth_clone_{i:03d}", media_path=case["prompt"], ground_truth=gt, + meta={"lang": "zh", "source": "synthetic", "duration_sec": self.duration, + "target_text": case["text"]}, + )) + return SampleManifest(dataset=self.name, samples=samples, meta={"generated": True}) diff --git a/src/translip_lab/metrics/__init__.py b/src/translip_lab/metrics/__init__.py index 0264716..5f11911 100644 --- a/src/translip_lab/metrics/__init__.py +++ b/src/translip_lab/metrics/__init__.py @@ -5,6 +5,7 @@ from .detection import box_iou, match_boxes, prf from .diarization import der, parse_rttm from .image import psnr, ssim +from .speaker import cosine_similarity, speaker_similarity from .text import cer, edit_distance, wer __all__ = [ @@ -13,4 +14,5 @@ "si_sdr", "sdr", "psnr", "ssim", "box_iou", "match_boxes", "prf", + "cosine_similarity", "speaker_similarity", ] diff --git a/src/translip_lab/metrics/speaker.py b/src/translip_lab/metrics/speaker.py new file mode 100644 index 0000000..c62aa9d --- /dev/null +++ b/src/translip_lab/metrics/speaker.py @@ -0,0 +1,85 @@ +"""Speaker-similarity (SIM) for voice-clone TTS eval: cosine over speaker embeddings. + +The ``tts-clone`` scenario asks "does the synthesized speech preserve the target +*timbre*" — the dimension translip's intrinsic dub score under-reports. Following +seed-tts-eval, SIM is the cosine similarity between a speaker embedding of the +reference voice and of the generated audio. + +The default embedder reuses translip's in-tree ECAPA (speechbrain) — the same +model the diarization backend uses — imported lazily so the lab stays importable +without it, and degrading to ``sim=None`` (never a crash) when it is absent. The +embedder is injectable so the metric is unit-testable offline with a pure-numpy +stand-in (no torch, no model download). +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +import numpy as np + +EmbedFn = Callable[[Any], "np.ndarray | None"] + + +def cosine_similarity(a: Any, b: Any, *, eps: float = 1e-12) -> float: + """Cosine similarity of two vectors, in [-1, 1]. Length-mismatch is truncated.""" + va = np.asarray(a, dtype=np.float64).reshape(-1) + vb = np.asarray(b, dtype=np.float64).reshape(-1) + n = min(len(va), len(vb)) + if n == 0: + return 0.0 + va, vb = va[:n], vb[:n] + denom = (float(np.linalg.norm(va)) * float(np.linalg.norm(vb))) + eps + return float(np.dot(va, vb) / denom) + + +def _ecapa_embedder(device: str = "auto") -> EmbedFn: + """Resolve translip's in-tree ECAPA embedder (lazy; may raise on missing deps). + + Loads the speechbrain classifier once and returns a ``path -> embedding`` + closure. Same model/weights the ``ecapa`` diarization backend uses, so SIM is + measured with the speaker space translip itself reasons in. + """ + from translip.speaker_embedding import ( + embedding_for_clip, + load_speechbrain_classifier, + read_audio_mono, + resolve_speaker_device, + ) + + classifier = load_speechbrain_classifier(resolve_speaker_device(device)) + + def embed(path: Any) -> "np.ndarray | None": + waveform, sample_rate = read_audio_mono(Path(path)) + return embedding_for_clip(classifier, waveform, sample_rate) + + return embed + + +def speaker_similarity( + reference_wav: Any, + hypothesis_wav: Any, + *, + embed_fn: EmbedFn | None = None, + device: str = "auto", +) -> dict[str, Any]: + """Cosine SIM between reference-voice and generated-audio speaker embeddings. + + Returns ``{"sim": float|None, "embedding_dim": int|None, "note"?: str}``. On any + embedder failure (speechbrain missing, model load error, empty/short clip) the + metric degrades to ``sim=None`` with a note rather than raising — the scenario + still reports intelligibility (CER). ``embed_fn`` overrides the default ECAPA + embedder (used by tests to stay offline). + """ + try: + embed = embed_fn or _ecapa_embedder(device) + ref_emb = embed(reference_wav) + hyp_emb = embed(hypothesis_wav) + except Exception as exc: # noqa: BLE001 — degrade, never crash the run + return {"sim": None, "embedding_dim": None, + "note": f"embedder unavailable: {type(exc).__name__}: {exc}"} + if ref_emb is None or hyp_emb is None: + return {"sim": None, "embedding_dim": None, + "note": "embedder returned no embedding (clip too short or silent?)"} + dim = int(np.asarray(ref_emb).reshape(-1).shape[0]) + return {"sim": cosine_similarity(ref_emb, hyp_emb), "embedding_dim": dim} diff --git a/src/translip_lab/scenarios/__init__.py b/src/translip_lab/scenarios/__init__.py index 2fbea7e..d5ff9d6 100644 --- a/src/translip_lab/scenarios/__init__.py +++ b/src/translip_lab/scenarios/__init__.py @@ -4,6 +4,6 @@ from ..core.scenario import available_scenarios, get_scenario, register_scenario # Importing each module registers its scenario. -from . import asr, diarization, e2e_dub, ocr_detect, separation, subtitle_erase # noqa: E402,F401 +from . import asr, diarization, e2e_dub, ocr_detect, separation, subtitle_erase, tts_clone # noqa: E402,F401 __all__ = ["get_scenario", "register_scenario", "available_scenarios"] diff --git a/src/translip_lab/scenarios/tts_clone.py b/src/translip_lab/scenarios/tts_clone.py new file mode 100644 index 0000000..15f7c94 --- /dev/null +++ b/src/translip_lab/scenarios/tts_clone.py @@ -0,0 +1,105 @@ +"""Voice-clone TTS scenario: synthesize target text in a reference voice, then score +timbre preservation (SIM) + intelligibility (CER) against ground truth. + +Fills the lab's biggest gap — the dubbing/TTS stage had no GT-anchored metric, only +``e2e-dub``'s intrinsic honest score (which under-reports the timbre band). Follows +seed-tts-eval: SIM via speaker-embedding cosine, intelligibility via ASR +re-transcription. **SIM is the primary metric** — it is exactly the timbre +dimension nothing else in the lab measures and the intrinsic score hides. + +invoke: ``tts_synth`` worker (clone) → ``translip transcribe`` (re-transcribe the synth). +score: CER/WER(reference text, re-transcript) + SIM(reference voice, synth). +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from ..core.invoke import Invoker, StageResult +from ..core.sample import Sample +from ..core.scenario import Scenario, register_scenario +from ..metrics.speaker import speaker_similarity +from ..metrics.text import cer as cer_metric +from ..metrics.text import wer as wer_metric + + +class TtsCloneScenario(Scenario): + name = "tts-clone" + primary_metric_key = "sim" + higher_is_better = True + + def required_gt(self) -> list[str]: + return ["clone_text"] + + def _reference_wav(self, sample: Sample) -> Path: + return Path(sample.ground_truth.clone_ref_wav or sample.media_path) + + def input_paths(self, sample: Sample) -> list[str | Path]: + return [sample.media_path, self._reference_wav(sample)] + + def invoke(self, sample, work_dir, invoker, *, config, timeout, log_path) -> StageResult: + text = sample.ground_truth.clone_text or "" + ref = self._reference_wav(sample) + lang = config.get("language") or sample.meta.get("lang") or "zh" + synth = work_dir / "synth.wav" + r_tts = invoker.module( + "translip_lab.tts_synth", + ["--text", text, "--reference", str(ref), "--output", str(synth), + "--language", str(lang), "--backend", str(config.get("tts_backend", "qwen3tts"))], + timeout=timeout, log_path=work_dir / "tts.log", + ) + if not r_tts.ok: + return r_tts + synth_path = r_tts.outputs.get("synth_wav", str(synth)) + asr_args = ["--input", synth_path, "--output-dir", str(work_dir / "transcribe"), + "--language", str(lang)] + for flag, key in (("--asr-backend", "asr_backend"), ("--asr-model", "asr_model"), + ("--device", "device")): + if config.get(key): + asr_args += [flag, str(config[key])] + r_asr = invoker.translip("transcribe", asr_args, timeout=timeout, log_path=log_path) + # Carry the synth path into the scored outputs even though ASR didn't emit it. + r_asr.outputs = {**r_asr.outputs, "synth_wav": synth_path} + return r_asr + + def score(self, sample, work_dir, stage, config) -> dict[str, Any]: + outputs = (stage.outputs or {}) if stage else {} + synth = outputs.get("synth_wav") + if not synth or not Path(synth).is_file(): + raise RuntimeError("tts-clone produced no readable synth_wav") + seg_path = outputs.get("segments") + if not seg_path or not Path(seg_path).is_file(): + raise RuntimeError("transcribe of the synth produced no readable segments JSON") + segments = json.loads(Path(seg_path).read_text(encoding="utf-8")).get("segments", []) + hyp_text = "".join(str(s.get("text", "")) for s in segments) + target = sample.ground_truth.clone_text or "" + + cer_value = cer_metric(target, hyp_text) + result: dict[str, Any] = { + "cer": round(cer_value, 4), + "wer": round(wer_metric(target, hyp_text), 4), + "intelligibility": round(1.0 - min(cer_value, 1.0), 4), + "reference_char_count": len("".join(target.split())), + } + sim = speaker_similarity(self._reference_wav(sample), synth, + device=str(config.get("device", "auto"))) + result["sim"] = round(sim["sim"], 4) if isinstance(sim.get("sim"), (int, float)) else None + if sim.get("note"): + result["sim_note"] = sim["note"] + return result + + def corpus_metrics(self, metrics_list): + out: dict[str, Any] = {} + total_ref = sum((m.get("reference_char_count") or 0) for m in metrics_list) + if total_ref > 0: + total_edits = sum((m.get("cer") or 0.0) * (m.get("reference_char_count") or 0) + for m in metrics_list) + out["cer_micro"] = round(total_edits / total_ref, 4) + sims = [m["sim"] for m in metrics_list if isinstance(m.get("sim"), (int, float))] + if sims: + out["sim_mean"] = round(sum(sims) / len(sims), 4) + return out + + +register_scenario(TtsCloneScenario()) diff --git a/src/translip_lab/suites/asr-diar-ramc.toml b/src/translip_lab/suites/asr-diar-ramc.toml new file mode 100644 index 0000000..aaa173a --- /dev/null +++ b/src/translip_lab/suites/asr-diar-ramc.toml @@ -0,0 +1,29 @@ +# MagicData-RAMC (OpenSLR SLR123): spontaneous 2-party Mandarin conversation with +# manual transcripts + per-speaker timestamps → CER (asr) and DER (diarization) from +# ONE corpus. Extends the lab's open CER/DER GT from meetings into the dialogue +# domain (closer to film/TV dialogue than AISHELL-4 / AliMeeting). +# +# Place data first: extract the OpenSLR SLR123 archive (~15G) under +# /magicdata-ramc/test/ (License: CC BY-NC-ND 4.0, academic/eval only). +# collar 0.25 + expected_speakers 2 match the official DER 7.96% baseline. +name = "asr-diar-ramc" +dataset = "magicdata-ramc" +scenarios = ["asr", "diarization"] +limit = 5 +timeout_sec = 3600 + +[dataset_params] +subset = "test" + +[scenario_config.asr] +asr_backend = "funasr" +asr_model = "paraformer-zh" +language = "zh" + +[scenario_config.diarization] +diarizer_backend = "ecapa" +asr_backend = "funasr" +asr_model = "paraformer-zh" +expected_speakers = 2 +collar = 0.25 +ignore_overlap = true diff --git a/src/translip_lab/suites/tts-clone-synthetic.toml b/src/translip_lab/suites/tts-clone-synthetic.toml new file mode 100644 index 0000000..48c2d23 --- /dev/null +++ b/src/translip_lab/suites/tts-clone-synthetic.toml @@ -0,0 +1,18 @@ +# Voice-clone TTS on self-built synthetic GT: clone a reference voice saying a +# target line, then score timbre (SIM) + intelligibility (CER). The synthetic +# "speakers" validate the plumbing + metrics offline; for real numbers feed real +# voice samples via the `folder` dataset (.clone.txt + optional .ref.wav). +# Real runs need a TTS backend (qwen3tts) and the ECAPA (speechbrain) embedder for SIM. +name = "tts-clone-synthetic" +dataset = "synthetic-clone" +scenarios = ["tts-clone"] +limit = 2 +timeout_sec = 1800 + +[dataset_params] +clips = 2 +duration = 4.0 + +[scenario_config.tts-clone] +tts_backend = "qwen3tts" +language = "zh" diff --git a/src/translip_lab/tts_synth.py b/src/translip_lab/tts_synth.py new file mode 100644 index 0000000..b5e398f --- /dev/null +++ b/src/translip_lab/tts_synth.py @@ -0,0 +1,87 @@ +"""Lab-side voice-clone TTS worker (run via ``python -m translip_lab.tts_synth``). + +Reuses translip's in-tree Qwen3-TTS voice-clone core *directly* — NOT the server +``atomic_tools`` wrapper, whose ``adapters/__init__`` eager-imports every heavy +adapter (demucs / vision / …) and would violate the lab's "base deps only" +principle. Runs in an isolated subprocess so the multi-GB TTS model is freed on +exit, exactly like the other lab stages and the orchestrator. Prints the +machine-readable ``key=value`` lines the lab ``Invoker`` parses:: + + synth_wav=/abs/path/synth.wav + sample_rate=24000 + tts_backend=qwen3tts + +Only ``qwen3tts`` is wired in v1 (the cleanest path: ``model.generate_voice_clone``). +``moss-tts-nano-onnx`` / ``voxcpm2`` need the pipeline's reference-audio +preprocessing (``ReferencePackage``) and a real-model smoke test — see +``docs/translip-lab-tts-clone-eval.zh-CN.md`` §5/§9. +""" +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Sequence + +_QWEN_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" + + +def synth_qwen_clone(*, text: str, reference: Path, language: str): + """Clone ``reference``'s voice saying ``text`` → (waveform, sample_rate). + + Mirrors the atomic TTS tool's ``_generate_voice_clone`` but imports only + ``translip.dubbing`` core (one-way dependency, no server side-effects). + """ + from translip.dubbing.backend import SynthSegmentInput, resolve_tts_device + from translip.dubbing.qwen_tts_backend import ( + _language_name, + _load_qwen_model, + _max_new_tokens_for, + _normalize_waveform, + ) + + device = resolve_tts_device("auto") + model = _load_qwen_model(_QWEN_CLONE_MODEL, device) + segment = SynthSegmentInput( + segment_id="lab-clone", speaker_id="lab", target_lang=language, target_text=text, + source_duration_sec=max(0.8, len(text) * 0.3), duration_budget_sec=None, + ) + wavs, sample_rate = model.generate_voice_clone( + text=text, language=_language_name(language), ref_audio=str(reference), + x_vector_only_mode=True, non_streaming_mode=True, + max_new_tokens=_max_new_tokens_for(segment), + ) + if not wavs: + raise RuntimeError("Qwen3-TTS returned no waveform for voice clone") + return _normalize_waveform(wavs[0]), int(sample_rate) + + +def main(argv: Sequence[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="lab voice-clone TTS worker") + ap.add_argument("--text", required=True) + ap.add_argument("--reference", required=True, help="reference voice wav to clone") + ap.add_argument("--output", required=True) + ap.add_argument("--language", default="zh") + ap.add_argument("--backend", default="qwen3tts") + args = ap.parse_args(argv) + + if args.backend != "qwen3tts": + raise SystemExit( + f"tts_synth v1 only supports qwen3tts; got {args.backend!r} " + "(moss/voxcpm2 wiring is a documented follow-up)" + ) + + import soundfile as sf + + waveform, sample_rate = synth_qwen_clone( + text=args.text, reference=Path(args.reference), language=args.language) + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + sf.write(out, waveform, sample_rate) + print(f"synth_wav={out}") + print(f"sample_rate={sample_rate}") + print(f"tts_backend={args.backend}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/lab/test_datasets_ramc.py b/tests/lab/test_datasets_ramc.py new file mode 100644 index 0000000..80124ef --- /dev/null +++ b/tests/lab/test_datasets_ramc.py @@ -0,0 +1,99 @@ +"""MagicData-RAMC parser + adapter: [start,end] speaker gender,dialect text → RTTM/SRT. + +The corpus can't be downloaded here, so these tests pin the *parsing logic* against a +fixture in the published RAMC transcript format. The parser is the one risk surface +(format built from published examples), so it gets the most coverage. +""" +from __future__ import annotations + +import numpy as np +import pytest +import soundfile as sf + +from translip_lab.config import load_config +from translip_lab.datasets.magicdata_ramc import MagicDataRamcDataset, parse_ramc_transcript +from translip_lab.metrics.diarization import parse_rttm + +# Three segments, two speakers, demographic attribute tokens present. +_SAMPLE = ( + "[1.319,6.691] G00000140 女,普通话 你好今天天气很好\n" + "[7.000,9.500] G00000141 男,四川话 是的我们出去走走吧\n" + "[10.000,12.000] G00000140 女,普通话 好的那走吧\n" +) + + +def test_parse_segments_speakers_and_text(tmp_path): + p = tmp_path / "conv.txt" + p.write_text(_SAMPLE, encoding="utf-8") + rows = parse_ramc_transcript(p) + assert len(rows) == 3 + assert rows[0] == (1.319, 6.691, "G00000140", "你好今天天气很好") + assert rows[1][2] == "G00000141" and rows[1][3] == "是的我们出去走走吧" + # the gender,dialect tokens must NOT leak into the reference text + joined = "".join(r[3] for r in rows) + assert "普通话" not in joined and "四川话" not in joined + + +def test_parse_handles_attrs_absent(tmp_path): + p = tmp_path / "c2.txt" + p.write_text("[0.0,2.0] C0 你好世界\n", encoding="utf-8") + assert parse_ramc_transcript(p) == [(0.0, 2.0, "C0", "你好世界")] + + +def test_parse_tolerates_spaced_brackets(tmp_path): + # a real-data variant: whitespace inside the [start , end] marker + p = tmp_path / "c4.txt" + p.write_text("[ 1.5 , 3.0 ] G9 男,普通话 喂你好\n", encoding="utf-8") + assert parse_ramc_transcript(p) == [(1.5, 3.0, "G9", "喂你好")] + + +def test_parse_preserves_comma_text_and_sorts_and_drops_empty(tmp_path): + p = tmp_path / "c3.txt" + # out-of-order; a zero-length segment; a text token that contains a comma but is + # NOT a gender,dialect attribute (must be preserved, not stripped). + p.write_text( + "[5.0,6.0] G2 后面说的话\n" + "[2.0,2.0] G1 空段\n" + "[1.0,3.0] G1 前面,还有逗号\n", + encoding="utf-8", + ) + rows = parse_ramc_transcript(p) + assert [r[0] for r in rows] == [1.0, 5.0] # sorted, zero-length dropped + assert rows[0][3] == "前面,还有逗号" # comma-bearing text kept intact + + +def test_adapter_builds_cer_and_der_gt(tmp_path, monkeypatch): + monkeypatch.setenv("TRANSLIP_LAB_HOME", str(tmp_path)) + cfg = load_config() + base = cfg.datasets_dir / "magicdata-ramc" / "test" + base.mkdir(parents=True) + (base / "conv1.txt").write_text(_SAMPLE, encoding="utf-8") + (base / "SPKINFO.txt").write_text("C0 G00000140 F ...\n", encoding="utf-8") # metadata → must be ignored + sf.write(base / "conv1.wav", np.zeros(1600, dtype=np.float32), 16000) + + manifest = MagicDataRamcDataset(cfg, subset="test").normalize() + assert manifest.dataset == "magicdata-ramc" and len(manifest) == 1 + sample = manifest.samples[0] + assert sample.sample_id == "conv1" + + # ASR GT (SRT): text present, attribute tokens stripped + srt_path = sample.ground_truth.transcript_srt + assert srt_path and srt_path.is_file() + srt = srt_path.read_text(encoding="utf-8") + assert "你好今天天气很好" in srt and "普通话" not in srt + + # diarization GT (RTTM): two speakers, three turns, durations correct + rttm_path = sample.ground_truth.rttm + assert rttm_path and rttm_path.is_file() + ref = parse_rttm(rttm_path) + assert len(ref) == 3 + assert {spk for _, _, spk in ref} == {"G00000140", "G00000141"} + start, end, _ = ref[0] + assert abs((end - start) - (6.691 - 1.319)) < 1e-3 + + +def test_adapter_missing_subset_raises(tmp_path, monkeypatch): + monkeypatch.setenv("TRANSLIP_LAB_HOME", str(tmp_path)) + cfg = load_config() + with pytest.raises(FileNotFoundError): + MagicDataRamcDataset(cfg, subset="nope").normalize() diff --git a/tests/lab/test_datasets_synthetic_clone.py b/tests/lab/test_datasets_synthetic_clone.py new file mode 100644 index 0000000..c40b0b5 --- /dev/null +++ b/tests/lab/test_datasets_synthetic_clone.py @@ -0,0 +1,43 @@ +"""synthetic-clone generator: deterministic (reference voice, target text) cases.""" +from __future__ import annotations + +import numpy as np +import soundfile as sf + +from translip_lab.config import load_config +from translip_lab.datasets.synthetic_clone import ( + SyntheticCloneDataset, + generate_clone_case, + synth_voice, +) + + +def test_synth_voice_deterministic_and_speaker_separable(): + a1 = synth_voice(0, duration=1.0, sr=16000) + a2 = synth_voice(0, duration=1.0, sr=16000) + b = synth_voice(3, duration=1.0, sr=16000) + assert np.array_equal(a1, a2) # same seed → identical waveform + # same-speaker pair is closer than a different-speaker pair + assert np.mean((a1 - a2) ** 2) < np.mean((a1 - b[: len(a1)]) ** 2) + + +def test_generate_clone_case_writes_prompt_and_text(tmp_path): + case = generate_clone_case(tmp_path / "c0", index=0, speaker_seed=0, duration=1.0) + assert case["prompt"].is_file() + data, sr = sf.read(case["prompt"]) + assert sr == 16000 and len(data) > 0 + assert isinstance(case["text"], str) and case["text"] + + +def test_dataset_normalize_builds_clone_gt(tmp_path, monkeypatch): + monkeypatch.setenv("TRANSLIP_LAB_HOME", str(tmp_path)) + cfg = load_config() + manifest = SyntheticCloneDataset(cfg, clips=2, duration=1.0).normalize() + assert manifest.dataset == "synthetic-clone" and len(manifest) == 2 + sample = manifest.samples[0] + assert sample.media_path.is_file() + assert sample.ground_truth.clone_text # target text present + assert sample.ground_truth.clone_ref_wav == sample.media_path # prompt is the SIM anchor + # round-trips through the JSON-serializable form the runner persists + gt = sample.ground_truth.to_dict() + assert gt["clone_text"] and gt["clone_ref_wav"] diff --git a/tests/lab/test_metrics_speaker.py b/tests/lab/test_metrics_speaker.py new file mode 100644 index 0000000..494ad8d --- /dev/null +++ b/tests/lab/test_metrics_speaker.py @@ -0,0 +1,48 @@ +"""Speaker-similarity metric: cosine math + injectable embedder + graceful degrade.""" +from __future__ import annotations + +import numpy as np + +from translip_lab.metrics.speaker import cosine_similarity, speaker_similarity + + +def test_cosine_identical_orthogonal_opposite(): + a = np.array([1.0, 2.0, 3.0]) + assert abs(cosine_similarity(a, a) - 1.0) < 1e-9 + assert abs(cosine_similarity([1.0, 0.0], [0.0, 1.0])) < 1e-9 + assert abs(cosine_similarity([1.0, 0.0], [-1.0, 0.0]) + 1.0) < 1e-9 + + +def test_cosine_length_mismatch_truncates(): + assert abs(cosine_similarity([1.0, 1.0, 9.0], [1.0, 1.0]) - 1.0) < 1e-9 + + +def test_cosine_empty_is_zero(): + assert cosine_similarity([], []) == 0.0 + + +def test_speaker_similarity_with_injected_embedder(): + table = { + "a.wav": np.array([1.0, 0.0, 0.0]), + "b.wav": np.array([2.0, 0.0, 0.0]), # same direction as a → sim 1 + "c.wav": np.array([0.0, 1.0, 0.0]), # orthogonal → sim 0 + } + embed = lambda p: table[str(p)] # noqa: E731 + same = speaker_similarity("a.wav", "b.wav", embed_fn=embed) + diff = speaker_similarity("a.wav", "c.wav", embed_fn=embed) + assert same["sim"] > 0.99 and same["embedding_dim"] == 3 + assert abs(diff["sim"]) < 1e-9 + assert same["sim"] > diff["sim"] + + +def test_speaker_similarity_degrades_to_none_on_embedder_error(): + def boom(_path): + raise RuntimeError("no speechbrain") + + out = speaker_similarity("a.wav", "b.wav", embed_fn=boom) + assert out["sim"] is None and "note" in out + + +def test_speaker_similarity_none_when_embedding_missing(): + out = speaker_similarity("a.wav", "b.wav", embed_fn=lambda _p: None) + assert out["sim"] is None and "note" in out diff --git a/tests/lab/test_scenario_tts_clone.py b/tests/lab/test_scenario_tts_clone.py new file mode 100644 index 0000000..0cbdcf1 --- /dev/null +++ b/tests/lab/test_scenario_tts_clone.py @@ -0,0 +1,78 @@ +"""tts-clone scenario scoring: SIM + CER over fabricated stage outputs (fully offline). + +No TTS model and no real audio: a fake StageResult supplies the synth path + a +transcript, and the ECAPA embedder is monkeypatched to a pure-numpy stand-in, so +the whole score path (timbre SIM + intelligibility CER) is exercised deterministically. +""" +from __future__ import annotations + +import json + +import numpy as np + +import translip_lab.metrics.speaker as speaker_mod +import translip_lab.scenarios # noqa: F401 — registers scenarios +from translip_lab.core.invoke import StageResult +from translip_lab.core.sample import GroundTruth, Sample +from translip_lab.core.scenario import SCENARIO_REGISTRY +from translip_lab.scenarios.tts_clone import TtsCloneScenario + + +def _stage(outputs): + return StageResult(argv=[], returncode=0, stdout="", stderr="", duration_sec=0.0, outputs=outputs) + + +def _write_segments(path, text): + path.write_text(json.dumps({"segments": [{"start": 0.0, "end": 2.0, "text": text}]}), encoding="utf-8") + + +def test_tts_clone_registered(): + assert "tts-clone" in SCENARIO_REGISTRY + + +def test_tts_clone_skips_without_target_text(tmp_path): + sample = Sample("s", tmp_path / "prompt.wav", GroundTruth()) # no clone_text + res = TtsCloneScenario().run(sample, tmp_path / "w", None, config={}) + assert res.status == "skipped" and "clone_text" in (res.error or "") + + +def test_tts_clone_perfect_vs_degraded(tmp_path, monkeypatch): + import soundfile as sf + + prompt = tmp_path / "prompt.wav" + synth_good = tmp_path / "synth_good.wav" + synth_bad = tmp_path / "synth_bad.wav" + for path in (prompt, synth_good, synth_bad): + sf.write(path, np.zeros(1600, dtype=np.float32), 16000) + + target = "你好世界今天天气很好" + sample = Sample("s", prompt, GroundTruth(clone_text=target, clone_ref_wav=prompt)) + scen = TtsCloneScenario() + + # Inject a pure-numpy embedder: prompt + good = same speaker, bad = different. + emb = { + str(prompt): np.array([1.0, 0.0, 0.0]), + str(synth_good): np.array([1.0, 0.0, 0.0]), + str(synth_bad): np.array([0.0, 1.0, 0.0]), + } + monkeypatch.setattr(speaker_mod, "_ecapa_embedder", lambda device="auto": (lambda p: emb[str(p)])) + + seg_good = tmp_path / "good.segments.json" + _write_segments(seg_good, target) # transcript == target + good = scen.score(sample, tmp_path, _stage({"synth_wav": str(synth_good), "segments": str(seg_good)}), {}) + assert good["cer"] < 0.01 and good["sim"] > 0.99 and good["intelligibility"] > 0.99 + + seg_bad = tmp_path / "bad.segments.json" + _write_segments(seg_bad, "完全不同的内容啊啊啊") # garbled transcript + bad = scen.score(sample, tmp_path, _stage({"synth_wav": str(synth_bad), "segments": str(seg_bad)}), {}) + assert bad["cer"] > good["cer"] and bad["sim"] < good["sim"] + + +def test_tts_clone_corpus_metrics(): + rows = [ + {"cer": 0.0, "reference_char_count": 10, "sim": 0.9}, + {"cer": 0.2, "reference_char_count": 10, "sim": 0.7}, + ] + corpus = TtsCloneScenario().corpus_metrics(rows) + assert abs(corpus["cer_micro"] - 0.1) < 1e-9 # pooled: (0*10 + 0.2*10) / 20 + assert abs(corpus["sim_mean"] - 0.8) < 1e-9 From 29db1ebf90b17be71f4e3cbb92530a648d5e40ba Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 11:03:42 +0800 Subject: [PATCH 05/10] feat(lab): surface tts-clone + MagicData-RAMC in the Lab UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lab server lists scenarios/datasets/suites dynamically, so the new tts-clone scenario, synthetic-clone / magicdata-ramc datasets, and their suites appear in live mode automatically. This wires up the rest: - ScenarioTag: render the tts-clone tag (音色克隆 / Voice Clone) and normalize hyphenated backend names (e2e-dub, tts-clone) to the underscore label keys, so live data renders localized tags too (not just mock) - leaderboard/regression: include `sim` (higher-is-better) in the primary-metric picker so tts-clone runs surface their SIM score - experiments: infer dataset/tags for the tts-clone-synthetic + asr-diar-ramc suites - labMock.ts: add the new scenario/datasets/suites + two demo runs so the offline demo represents the new capabilities - i18n: tagClone (zh/en) Verified in-browser (system Chrome via Playwright): mock and live modes both render the new datasets/suites/tag and the tts-clone run's SIM metric; live shows "实时数据" against the real :8799 server with 0 console errors. --- frontend/src/api/labMock.ts | 57 +++++++++++++++++++++++++++++- frontend/src/i18n/messages.ts | 2 ++ frontend/src/pages/lab/LabPage.tsx | 10 ++++-- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/frontend/src/api/labMock.ts b/frontend/src/api/labMock.ts index 71e8425..b7b7ec3 100644 --- a/frontend/src/api/labMock.ts +++ b/frontend/src/api/labMock.ts @@ -15,6 +15,7 @@ export const MOCK_SCENARIOS: LabScenario[] = [ { name: 'ocr_detect', primary_metric: 'f1', higher_is_better: true, required_gt: ['ocr_boxes'] }, { name: 'separation', primary_metric: 'si_sdr', higher_is_better: true, required_gt: ['stem_audio'] }, { name: 'e2e_dub', primary_metric: 'mcd', higher_is_better: false, required_gt: ['reference_dub'] }, + { name: 'tts-clone', primary_metric: 'sim', higher_is_better: true, required_gt: ['clone_text'] }, ] export const MOCK_SUITES: string[] = [ @@ -26,6 +27,8 @@ export const MOCK_SUITES: string[] = [ 'subtitle-erase-synthetic', 'ocr-detect-synthetic', 'e2e-dub-folder', + 'tts-clone-synthetic', + 'asr-diar-ramc', ] export const MOCK_DATASETS: LabDataset[] = [ @@ -62,6 +65,17 @@ export const MOCK_DATASETS: LabDataset[] = [ samples: 50, total_duration_min: 180, }, + { + name: 'magicdata-ramc', + license: 'CC BY-NC-ND 4.0 (SLR123)', + provides: ['asr (CER)', 'diarization (DER)'], + subset: 'test', + subset_root: '/Users/lab/datasets/magicdata-ramc/test', + subset_exists: false, + expected_layout: 'magicdata-ramc//**/*.wav + .txt', + samples: 43, + total_duration_min: 1238, + }, { name: 'synthetic-subtitle', license: 'CC0', @@ -84,6 +98,17 @@ export const MOCK_DATASETS: LabDataset[] = [ samples: 24, total_duration_min: 0.8, }, + { + name: 'synthetic-clone', + license: 'CC0', + provides: ['tts-clone (SIM)'], + subset: 'mini', + subset_root: '/Users/lab/cache/synthetic-clone/mini', + subset_exists: true, + expected_layout: 'auto-generated at runtime', + samples: 2, + total_duration_min: 0.13, + }, { name: 'folder', license: 'user-provided', @@ -170,6 +195,36 @@ export const MOCK_RUNS: LabRunSummary[] = [ }, arm_label: 'mdx-net', }, + { + run_id: '20260623-1012-tts-clone-synthetic', + suite: 'tts-clone-synthetic', + dataset: 'synthetic-clone', + scenarios: ['tts-clone'], + status: 'finished', + created_at: '2026-06-23 10:12:30', + duration_sec: 96, + num_samples: 2, + aggregates: { + 'tts-clone': { sim: 0.812, cer_micro: 0.104, samples: 2 }, + }, + arm_label: 'qwen3tts', + notes: 'voice-clone SIM + intelligibility', + }, + { + run_id: '20260623-0930-asr-diar-ramc', + suite: 'asr-diar-ramc', + dataset: 'magicdata-ramc', + scenarios: ['asr', 'diarization'], + status: 'finished', + created_at: '2026-06-23 09:30:14', + duration_sec: 540, + num_samples: 5, + aggregates: { + asr: { cer_micro: 0.191, rtf: 0.21 }, + diarization: { der: 0.0796, jer: 0.102 }, + }, + arm_label: 'paraformer-zh + ecapa', + }, { run_id: '20260622-0805-asr-drama-paraformer-rerun', suite: 'asr-drama-wenetspeech', @@ -224,7 +279,7 @@ const _runById = new Map(MOCK_RUNS.map(r => [r.run_id, r])) function pickPrimary(run: LabRunSummary | undefined, scenario = 'asr'): number | null { if (!run) return null const m = run.aggregates?.[scenario] ?? {} - const v = m.cer_micro ?? m.cer ?? m.der ?? m.si_sdr ?? m.f1 ?? m.psnr ?? m.mcd + const v = m.sim ?? m.cer_micro ?? m.cer ?? m.der ?? m.si_sdr ?? m.f1 ?? m.psnr ?? m.mcd return typeof v === 'number' ? v : null } diff --git a/frontend/src/i18n/messages.ts b/frontend/src/i18n/messages.ts index 0ef0e6d..1197863 100644 --- a/frontend/src/i18n/messages.ts +++ b/frontend/src/i18n/messages.ts @@ -2169,6 +2169,7 @@ const zhMessages = { tagOcr: 'OCR', tagErase: '字幕擦除', tagDub: '配音端到端', + tagClone: '音色克隆', }, leaderboard: { heading: '历史 Run', @@ -4401,6 +4402,7 @@ const enMessages: LocaleMessages = { tagOcr: 'OCR', tagErase: 'Erase', tagDub: 'End-to-end Dub', + tagClone: 'Voice Clone', }, leaderboard: { heading: 'Past runs', diff --git a/frontend/src/pages/lab/LabPage.tsx b/frontend/src/pages/lab/LabPage.tsx index 1af6a23..1b86624 100644 --- a/frontend/src/pages/lab/LabPage.tsx +++ b/frontend/src/pages/lab/LabPage.tsx @@ -107,8 +107,11 @@ function ScenarioTag({ name }: { name: string }) { ocr_detect: { label: t.lab.experiments.tagOcr, cls: 'bg-emerald-50 text-emerald-700' }, subtitle_erase: { label: t.lab.experiments.tagErase, cls: 'bg-pink-50 text-pink-700' }, e2e_dub: { label: t.lab.experiments.tagDub, cls: 'bg-amber-50 text-amber-700' }, + tts_clone: { label: t.lab.experiments.tagClone, cls: 'bg-sky-50 text-sky-700' }, } - const m = map[name] ?? { label: name, cls: 'bg-[#f3f4f6] text-[#6b7280]' } + // backend scenario names are hyphenated (e2e-dub, tts-clone); normalize to the + // underscore keys so both live and mock data render a localized tag. + const m = map[name.replace(/-/g, '_')] ?? { label: name, cls: 'bg-[#f3f4f6] text-[#6b7280]' } return ( {m.label} @@ -132,7 +135,7 @@ function StatCard({ icon: Icon, label, value, hint }: { icon: typeof Database; l function pickPrimaryMetric(run: LabRunSummary): { scenario: string; key: string; value: number } | null { const aggregates = run.aggregates ?? {} for (const [scenario, metrics] of Object.entries(aggregates)) { - for (const k of ['cer_micro', 'cer', 'der', 'mcd', 'si_sdr', 'f1', 'psnr']) { + for (const k of ['sim', 'cer_micro', 'cer', 'der', 'mcd', 'si_sdr', 'f1', 'psnr']) { const v = metrics[k] if (typeof v === 'number') return { scenario, key: k, value: v } } @@ -299,13 +302,16 @@ function ExperimentsTab() { if (suite.includes('ocr')) tags.push('ocr_detect') if (suite.includes('erase')) tags.push('subtitle_erase') if (suite.includes('dub')) tags.push('e2e_dub') + if (suite.includes('tts') || suite.includes('clone')) tags.push('tts_clone') return tags } function inferDataset(suite: string): string { if (suite.includes('wenetspeech')) return 'wenetspeech-drama' + if (suite.includes('ramc')) return 'magicdata-ramc' if (suite.includes('aishell4')) return 'aishell4' if (suite.includes('alimeeting')) return 'alimeeting' + if (suite.includes('clone')) return 'synthetic-clone' if (suite.includes('synthetic-mix')) return 'synthetic-mix' if (suite.includes('synthetic')) return 'synthetic-subtitle' return 'folder' From 73852d14605ed82b234621937e9e0a4bf998c4e8 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 11:05:05 +0800 Subject: [PATCH 06/10] docs(lab): record tts-clone/RAMC UI surfacing + browser verification --- docs/translip-lab-tts-clone-eval.zh-CN.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/translip-lab-tts-clone-eval.zh-CN.md b/docs/translip-lab-tts-clone-eval.zh-CN.md index 0f7cb93..42c8c51 100644 --- a/docs/translip-lab-tts-clone-eval.zh-CN.md +++ b/docs/translip-lab-tts-clone-eval.zh-CN.md @@ -190,3 +190,18 @@ from translip.dubbing.qwen_tts_backend import ( 3. **ChaLearn Decaptioning** 适配器（subtitle-erase 跨集校验；调研 P1）。 4. **缓存键纳入 scorer 版本**（`cache.py` 现仅按 config+input 哈希，改打分逻辑不失效——既有 footgun；加 `Scenario.version` 进 key）。 5. 自建中文影视音色小集（走 §3.3 的 `folder` 边车）。 + +--- + +## 10. 前端可见性与浏览器验证（已完成） + +lab 服务端的 scenarios/datasets/suites 均按注册表**动态列出**，故新增的 `tts-clone` 场景、`synthetic-clone`/`magicdata-ramc` 数据集、`tts-clone-synthetic`/`asr-diar-ramc` 套件在**实时模式自动出现**。前端（`frontend/src/pages/lab/LabPage.tsx` · `api/labMock.ts` · `i18n/messages.ts`）补齐： +- `ScenarioTag` 渲染「音色克隆 / Voice Clone」标签，并把连字符后端名（`e2e-dub`/`tts-clone`）归一到下划线标签键 —— 实时数据也能本地化渲染标签； +- 排行榜/回归把 `sim`（越大越好）纳入主指标选取，tts-clone run 能展示 SIM； +- 实验页为新套件推断数据集与标签； +- `labMock.ts` 离线演示补齐新场景/数据集/套件 + 两条示例 run；i18n 加 `tagClone`（中英）。 + +**浏览器验证**（系统 Chrome via Playwright，覆盖 mock + live 两模式）： +- mock（lab 服务端关闭→兜底演示数据）：四个 Tab 均渲染新数据集/套件/「音色克隆」标签，排行榜显示 tts-clone run 的 `sim=0.812`； +- live（lab 服务端 :8799）：徽标显示「实时数据」、0 控制台错误，数据集表显示 `magicdata-ramc` 的真实 describe 元数据（license / 提供指标 / `[start,end] speaker 性别,方言 text` 布局）； +- 前端 `npm run build`（tsc 全量类型检查）通过、`vitest` 189 测试通过、ESLint 0 报错。 From 631aec648bbfd2b5056b9131ff0fc46961e7f0f6 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 11:11:04 +0800 Subject: [PATCH 07/10] fix(lab): key the result cache on scenario.version (close stale-score footgun) The cache was keyed only by config + input fingerprints, so changing a scenario's invoke/score logic (or a metric it uses) without touching config/inputs silently served stale cached scores. Add Scenario.version (default 1) into the cache key; bump it to invalidate. Also recorded in each run's scenario_meta for traceability. --no-cache still bypasses. Tests: scenario_cache_key version sensitivity + default-is-1; runner-level version-bump-invalidates-cache (tests/lab 107 passing). --- CLAUDE.md | 2 +- docs/translip-lab-tts-clone-eval.zh-CN.md | 2 +- src/translip_lab/core/cache.py | 17 +++++++++++------ src/translip_lab/core/runner.py | 4 +++- src/translip_lab/core/scenario.py | 4 ++++ tests/lab/test_core_cache.py | 17 +++++++++++++++++ tests/lab/test_core_runner.py | 13 +++++++++++++ 7 files changed, 50 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7010477..ff16c84 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,7 +111,7 @@ React 19 + TypeScript + Vite 8 + Tailwind 4. **Server state via TanStack React Q ### `translip_lab` evaluation lab (`src/translip_lab/`, optional extra `lab`) -A **loosely-coupled** harness that benchmarks existing translip capabilities against ground-truth datasets (CER/DER/SI-SDR/PSNR-SSIM/OCR-text-F1). **One-way dependency rule: `translip_lab` imports `translip`; translip never imports the lab.** Integration is via the stable CLI/JSON contract (subprocess, like the orchestrator) plus a couple of pure helper imports — deleting `src/translip_lab/` + the one "Testing Lab" sidebar link leaves translip untouched. Entry points `translip-lab` (CLI: `doctor`/`run`/`report`/`compare`) and `translip-lab-server` (standalone dashboard on `:8799`, its own origin; the main UI only links to it). Suites in `translip_lab/suites/*.toml` declare dataset+scenarios+config (+ `[[arms]]` for config sweeps). Data/runs default to `/Volumes/EXT/translip-lab` (`TRANSLIP_LAB_HOME`). The engine runs on base deps (numpy/scipy/soundfile) + ffmpeg + stdlib; only Pillow is added by the `lab` extra. See `src/translip_lab/README.md`. Caveat: the lab cache is keyed by config+input fingerprints, **not code** — re-run with `--no-cache` after changing scoring logic. +A **loosely-coupled** harness that benchmarks existing translip capabilities against ground-truth datasets (CER/DER/SI-SDR/PSNR-SSIM/OCR-text-F1). **One-way dependency rule: `translip_lab` imports `translip`; translip never imports the lab.** Integration is via the stable CLI/JSON contract (subprocess, like the orchestrator) plus a couple of pure helper imports — deleting `src/translip_lab/` + the one "Testing Lab" sidebar link leaves translip untouched. Entry points `translip-lab` (CLI: `doctor`/`run`/`report`/`compare`) and `translip-lab-server` (standalone dashboard on `:8799`, its own origin; the main UI only links to it). Suites in `translip_lab/suites/*.toml` declare dataset+scenarios+config (+ `[[arms]]` for config sweeps). Data/runs default to `/Volumes/EXT/translip-lab` (`TRANSLIP_LAB_HOME`). The engine runs on base deps (numpy/scipy/soundfile) + ffmpeg + stdlib; only Pillow is added by the `lab` extra. See `src/translip_lab/README.md`. Caveat: the lab cache is keyed by config+input fingerprints **plus each scenario's `version`** — bump `Scenario.version` to invalidate cached results when its invoke/score logic (or a metric it uses) changes; `--no-cache` still bypasses. ## Key paths & env diff --git a/docs/translip-lab-tts-clone-eval.zh-CN.md b/docs/translip-lab-tts-clone-eval.zh-CN.md index 42c8c51..216fefb 100644 --- a/docs/translip-lab-tts-clone-eval.zh-CN.md +++ b/docs/translip-lab-tts-clone-eval.zh-CN.md @@ -188,7 +188,7 @@ from translip.dubbing.qwen_tts_backend import ( 1. **moss/voxcpm2** 后端的 worker 路径 + 真实模型 smoke test。 2. **MagicData-RAMC** 适配器 — **✅ 已落地**：`datasets/magicdata_ramc.py`（专用 `[start,end] speaker 性别,方言 text` 解析器，复用 RTTM/SRT 发射器）+ `suites/asr-diar-ramc.toml` + `tests/lab/test_datasets_ramc.py`（6 个 fixture 测试验证解析逻辑）。CER+DER 扩到自发对话域。⚠️ 解析格式据官方发布示例构建，**真数据首跑前抽一条 `.txt` 逐字节对照**（解析器已隔离便于微调）。 3. **ChaLearn Decaptioning** 适配器（subtitle-erase 跨集校验；调研 P1）。 -4. **缓存键纳入 scorer 版本**（`cache.py` 现仅按 config+input 哈希，改打分逻辑不失效——既有 footgun；加 `Scenario.version` 进 key）。 +4. **缓存键纳入 scorer 版本** — **✅ 已落地**：`Scenario.version`（默认 1）进 `scenario_cache_key`，改打分逻辑时 bump 即让旧缓存失效（`--no-cache` 仍可绕过），并写入每个 run 的 `scenario_meta` 便于溯源；含单测（键敏感性 + 默认值）与集成测试（bump version → 缓存 miss → 重跑）。 5. 自建中文影视音色小集（走 §3.3 的 `folder` 边车）。 --- diff --git a/src/translip_lab/core/cache.py b/src/translip_lab/core/cache.py index 6bc59f1..569a219 100644 --- a/src/translip_lab/core/cache.py +++ b/src/translip_lab/core/cache.py @@ -1,9 +1,12 @@ -"""Cache keys for scenario results — SHA256 over (scenario, sample, config, inputs). +"""Cache keys for scenario results — SHA256 over (scenario, version, sample, config, inputs). -Mirrors the orchestrator's idea: a result is reusable only if the scenario, the -sample, the scenario config, and the input file fingerprints are all unchanged. -File fingerprints use size+mtime (cheap; media can be multi-GB) rather than -content hashing. +Mirrors the orchestrator's idea: a result is reusable only if the scenario, its +scorer ``version``, the sample, the scenario config, and the input file +fingerprints are all unchanged. ``version`` closes a footgun — changing a +scenario's invoke/score logic (or a metric it uses) *without* touching config or +inputs would otherwise serve stale cached scores; bumping ``Scenario.version`` +invalidates them. File fingerprints use size+mtime (cheap; media can be multi-GB) +rather than content hashing. """ from __future__ import annotations @@ -21,10 +24,12 @@ def fingerprint_path(path: str | Path) -> str: return f"{p}:MISSING" -def scenario_cache_key(*, scenario: str, sample_id: str, config: dict, input_paths: list[str | Path]) -> str: +def scenario_cache_key(*, scenario: str, sample_id: str, config: dict, input_paths: list[str | Path], + code_version: int = 1) -> str: payload = json.dumps( { "scenario": scenario, + "version": code_version, "sample": sample_id, "config": config, "inputs": sorted(fingerprint_path(p) for p in input_paths), diff --git a/src/translip_lab/core/runner.py b/src/translip_lab/core/runner.py index fc5e6e9..fe8e537 100644 --- a/src/translip_lab/core/runner.py +++ b/src/translip_lab/core/runner.py @@ -74,6 +74,7 @@ def run_suite( key = scenario_cache_key( scenario=scenario.name, sample_id=sample.sample_id, config=cfg, input_paths=[str(p) for p in scenario.input_paths(sample)], + code_version=scenario.version, ) cache_file = cache_results_dir / f"{key}.json" @@ -111,7 +112,8 @@ def run_suite( }) scenario_meta = { - s.name: {"primary_metric_key": s.primary_metric_key, "higher_is_better": s.higher_is_better} + s.name: {"primary_metric_key": s.primary_metric_key, "higher_is_better": s.higher_is_better, + "version": s.version} for s in scenarios } aggregates = summarize_aggregates(results, scenario_meta) diff --git a/src/translip_lab/core/scenario.py b/src/translip_lab/core/scenario.py index 5ddbdff..cc1ff25 100644 --- a/src/translip_lab/core/scenario.py +++ b/src/translip_lab/core/scenario.py @@ -70,6 +70,10 @@ class Scenario(ABC): name: str = "scenario" primary_metric_key: str = "score" higher_is_better: bool = True + # Scorer version — feeds the cache key. Bump when this scenario's invoke/score + # logic (or a metric it relies on) changes, so cached results from the old logic + # are invalidated instead of silently reused. + version: int = 1 def required_gt(self) -> list[str]: """GroundTruth attribute names that must be present for this scenario.""" diff --git a/tests/lab/test_core_cache.py b/tests/lab/test_core_cache.py index 4f44e7d..56e12b2 100644 --- a/tests/lab/test_core_cache.py +++ b/tests/lab/test_core_cache.py @@ -27,3 +27,20 @@ def test_key_changes_with_input_content(tmp_path): f.write_bytes(b"abcd") # size change → fingerprint change k2 = scenario_cache_key(scenario="asr", sample_id="s", config={}, input_paths=[f]) assert k1 != k2 + + +def test_key_changes_with_scorer_version(tmp_path): + # the footgun fix: same config + inputs but a bumped scorer version must not collide + f = tmp_path / "x.wav" + f.write_bytes(b"abc") + k1 = scenario_cache_key(scenario="asr", sample_id="s", config={}, input_paths=[f], code_version=1) + k2 = scenario_cache_key(scenario="asr", sample_id="s", config={}, input_paths=[f], code_version=2) + assert k1 != k2 + + +def test_version_defaults_to_one(tmp_path): + f = tmp_path / "x.wav" + f.write_bytes(b"abc") + explicit = scenario_cache_key(scenario="asr", sample_id="s", config={}, input_paths=[f], code_version=1) + default = scenario_cache_key(scenario="asr", sample_id="s", config={}, input_paths=[f]) + assert explicit == default diff --git a/tests/lab/test_core_runner.py b/tests/lab/test_core_runner.py index ab7b090..3f2b3b0 100644 --- a/tests/lab/test_core_runner.py +++ b/tests/lab/test_core_runner.py @@ -72,6 +72,19 @@ def test_caching_skips_second_invocation(tmp_path): assert sc.invocations == 1 # second run hit the cache → no new invocation +def test_cache_invalidated_by_version_bump(tmp_path): + cfg = _cfg(tmp_path) + m = _manifest(tmp_path) + sc = CountingScenario() + run_suite(manifest=m, scenarios=[sc], suite="s", invoker=FakeInvoker(), + lab_config=cfg, use_cache=True, run_id="r1") + assert sc.invocations == 1 + sc.version = 2 # e.g. the scoring logic changed → cached r1 result must not be reused + run_suite(manifest=m, scenarios=[sc], suite="s", invoker=FakeInvoker(), + lab_config=cfg, use_cache=True, run_id="r2") + assert sc.invocations == 2 # cache miss → re-invoked under the new version + + def test_skipped_when_missing_ground_truth(tmp_path): cfg = _cfg(tmp_path) From 3440531c0b8286405000595446334ab59d83580b Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 11:35:46 +0800 Subject: [PATCH 08/10] feat(lab): run detail page + failure traceability + report download (PRD P0-1/2/4) Leaderboard rows now link to a per-run detail view the lab server already had data for (GET /api/lab/runs/{id}) but the UI never surfaced. - RunDetailPage: run header, meta cards (samples/scenarios/started/elapsed), aggregates table (primary metric / mean / micro / std / scored-failed-skipped), and a per-sample results table. Reads the real run-manifest shape with defensive fallbacks to the mock summary shape. - P0-2 failure traceability: failed samples show an expandable stderr/traceback panel. - P0-4 report download: GET /api/lab/runs/{id}/report.md (run_to_markdown) + a download button (live fetch; mock builds the markdown client-side). - labMock: enriched run details (full-shape aggregates + per-sample results incl. a failure) so the offline demo represents the page; i18n detail block (zh/en). Tests: backend report endpoint (tests/lab 108 passing); frontend build + 189 vitest + eslint clean. Browser-verified (system Chrome) in mock AND live (fabricated real manifest) modes: detail renders, error expands, 0 console errors live. --- frontend/src/App.tsx | 4 + frontend/src/api/lab.ts | 6 + frontend/src/api/labMock.ts | 90 +++++++-- frontend/src/i18n/messages.ts | 50 +++++ frontend/src/pages/lab/LabPage.tsx | 6 +- frontend/src/pages/lab/RunDetailPage.tsx | 239 +++++++++++++++++++++++ src/translip_lab/server/app.py | 16 +- tests/lab/test_server.py | 17 ++ 8 files changed, 413 insertions(+), 15 deletions(-) create mode 100644 frontend/src/pages/lab/RunDetailPage.tsx diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 893ceb8..415e3b5 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -70,6 +70,9 @@ const ApiDocsPage = lazy(() => const LabPage = lazy(() => import('./pages/lab/LabPage').then(module => ({ default: module.LabPage })), ) +const RunDetailPage = lazy(() => + import('./pages/lab/RunDetailPage').then(module => ({ default: module.RunDetailPage })), +) const queryClient = new QueryClient({ defaultOptions: { @@ -116,6 +119,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> diff --git a/frontend/src/api/lab.ts b/frontend/src/api/lab.ts index 8fba1eb..19d7353 100644 --- a/frontend/src/api/lab.ts +++ b/frontend/src/api/lab.ts @@ -6,6 +6,7 @@ import { MOCK_SCENARIOS, MOCK_SUITES, buildMockCompare, + buildMockReport, buildMockTriggerResponse, } from './labMock' @@ -170,6 +171,11 @@ export const labApi = { () => labClient.get(`/api/lab/runs/${runId}`).then(r => r.data), () => MOCK_RUN_DETAILS[runId] ?? { ...MOCK_RUNS[0], run_id: runId }, ), + reportMarkdown: (runId: string) => + callOrFallback( + () => labClient.get(`/api/lab/runs/${runId}/report.md`, { responseType: 'text' }).then(r => String(r.data)), + () => buildMockReport(runId), + ), compare: (baseline: string, candidate: string) => callOrFallback( () => diff --git a/frontend/src/api/labMock.ts b/frontend/src/api/labMock.ts index b7b7ec3..f252ae8 100644 --- a/frontend/src/api/labMock.ts +++ b/frontend/src/api/labMock.ts @@ -253,23 +253,65 @@ export const MOCK_RUNS: LabRunSummary[] = [ }, ] +function mockResults(r: LabRunSummary): Array> { + const scenarios = r.scenarios?.length ? r.scenarios : ['asr'] + const n = Math.max(3, Math.min(r.num_samples ?? 3, 4)) + const arm = String((r as Record).arm_label ?? 'default') + const rows: Array> = [] + for (let i = 0; i < n; i++) { + const scenario = scenarios[i % scenarios.length] + const agg = r.aggregates?.[scenario] ?? {} + const raw = agg.sim ?? agg.cer_micro ?? agg.der ?? agg.si_sdr ?? agg.cer + const base = typeof raw === 'number' ? raw : undefined + const failed = r.status === 'failed' && i === 0 // demo a traceable failure + rows.push({ + sample_id: `${r.dataset ?? 'sample'}_${String(i + 1).padStart(4, '0')}`, + scenario, + arm, + status: failed ? 'failed' : 'succeeded', + primary_metric: failed || base === undefined ? null : Number((base * (0.92 + 0.04 * i)).toFixed(4)), + duration_sec: 4 + i * 3, + cached: i === n - 1, + ...(failed ? { error: r.error ?? 'stage exit 1: RuntimeError: GPU OOM\n at sample 3/32' } : {}), + }) + } + return rows +} + +// Promote the leaderboard's flat {metric: value} aggregates into the full run-store +// aggregate shape (primary_metric / mean / scored…) the detail page (and live API) use. +function mockAggregates(r: LabRunSummary): Record> { + const arm = String((r as Record).arm_label ?? 'default') + const out: Record> = {} + for (const [scenario, metrics] of Object.entries(r.aggregates ?? {})) { + const def = MOCK_SCENARIOS.find(s => s.name === scenario) + const primary = def?.primary_metric ?? Object.keys(metrics)[0] ?? 'metric' + const mean = metrics[primary] + out[scenario] = { + scenario, + arm, + primary_metric: primary, + higher_is_better: def?.higher_is_better ?? false, + mean: typeof mean === 'number' ? mean : null, + std: 0, + scored: r.num_samples ?? 0, + failed: r.status === 'failed' ? 1 : 0, + skipped: 0, + } + } + return out +} + export const MOCK_RUN_DETAILS: Record = Object.fromEntries( MOCK_RUNS.map(r => [ r.run_id, { ...r, - manifest: { - dataset: r.dataset, - subset: 'mini', - license: 'WeNet Open Source — research only', - source: 'https://wenet.org.cn/WenetSpeech/', - drama_only: r.suite === 'asr-drama-wenetspeech', - }, - results: Array.from({ length: Math.min(r.num_samples ?? 0, 5) }).map((_, i) => ({ - sample_id: `Y0000000000_drama_${String(i + 1).padStart(4, '0')}`, - duration_sec: 4 + Math.random() * 6, - scenario_metrics: r.aggregates, - })), + aggregates: mockAggregates(r) as unknown as Record>, + sample_count: r.num_samples, + started_at: r.created_at, + elapsed_sec: r.duration_sec, + results: mockResults(r), }, ]), ) @@ -319,3 +361,27 @@ export function buildMockTriggerResponse(payload: LabTriggerRunPayload): LabTrig if (payload.no_cache) cmd.push('--no-cache') return { status: 'started', cmd } } + +export function buildMockReport(runId: string): string { + const r = MOCK_RUN_DETAILS[runId] ?? MOCK_RUNS.find(x => x.run_id === runId) + if (!r) return `# Lab run \`${runId}\`\n\n(not found — demo data)\n` + const lines = [ + `# Lab run \`${runId}\``, + '', + `- suite: \`${r.suite ?? '—'}\``, + `- dataset: \`${r.dataset ?? '—'}\``, + `- samples: ${(r as Record).sample_count ?? r.num_samples ?? '—'}`, + '', + '## Aggregates', + '', + '| scenario | metric | mean |', + '|---|---|---|', + ] + for (const [name, agg] of Object.entries(r.aggregates ?? {})) { + const a = agg as Record + const metric = ['cer_micro', 'sim', 'der', 'si_sdr', 'cer', 'psnr', 'f1'].find(k => k in a) ?? '—' + lines.push(`| ${name} | ${metric} | ${String(a[metric] ?? '—')} |`) + } + lines.push('', '_(demo data — generated client-side)_', '') + return lines.join('\n') +} diff --git a/frontend/src/i18n/messages.ts b/frontend/src/i18n/messages.ts index 1197863..a83e270 100644 --- a/frontend/src/i18n/messages.ts +++ b/frontend/src/i18n/messages.ts @@ -2121,6 +2121,31 @@ const zhMessages = { mock: '演示数据', mockHint: '后端未启动，当前展示的是 mock 数据，用于产品体验评审', }, + detail: { + back: '返回排行榜', + notFound: '未找到该 run', + downloadReport: '下载报告', + suite: '套件', + dataset: '数据集', + samples: '样本数', + scenarios: '场景', + started: '开始时间', + elapsed: '耗时', + aggregates: '聚合指标', + samplesTitle: '样本结果', + colScenario: '场景', + colMetric: '主指标', + colScored: '已评', + colFailed: '失败', + colSkipped: '跳过', + colSample: '样本', + colStatus: '状态', + colDuration: '耗时', + colCached: '缓存', + noSamples: '暂无样本结果', + showError: '查看错误', + hideError: '收起', + }, overview: { datasets: '已注册数据集', suites: '可用 Suite', @@ -4354,6 +4379,31 @@ const enMessages: LocaleMessages = { mock: 'Demo data', mockHint: 'Backend offline — showing mock data so you can review the product UX.', }, + detail: { + back: 'Back to leaderboard', + notFound: 'Run not found', + downloadReport: 'Download report', + suite: 'Suite', + dataset: 'Dataset', + samples: 'Samples', + scenarios: 'Scenarios', + started: 'Started', + elapsed: 'Elapsed', + aggregates: 'Aggregates', + samplesTitle: 'Per-sample results', + colScenario: 'Scenario', + colMetric: 'Primary', + colScored: 'Scored', + colFailed: 'Failed', + colSkipped: 'Skipped', + colSample: 'Sample', + colStatus: 'Status', + colDuration: 'Duration', + colCached: 'Cached', + noSamples: 'No per-sample results', + showError: 'Show error', + hideError: 'Hide', + }, overview: { datasets: 'Datasets', suites: 'Suites', diff --git a/frontend/src/pages/lab/LabPage.tsx b/frontend/src/pages/lab/LabPage.tsx index 1b86624..fad97c9 100644 --- a/frontend/src/pages/lab/LabPage.tsx +++ b/frontend/src/pages/lab/LabPage.tsx @@ -1,6 +1,6 @@ import { useMemo, useState, useSyncExternalStore } from 'react' import { useMutation, useQuery } from '@tanstack/react-query' -import { useSearchParams } from 'react-router-dom' +import { Link, useSearchParams } from 'react-router-dom' import { ArrowDownRight, ArrowUpRight, @@ -427,7 +427,9 @@ function LeaderboardTab() { {isBest ? : idx + 1} - {run.run_id} + + {run.run_id} + {run.suite ?? '—'} {String((run as Record).arm_label ?? '—')} diff --git a/frontend/src/pages/lab/RunDetailPage.tsx b/frontend/src/pages/lab/RunDetailPage.tsx new file mode 100644 index 0000000..b8df90c --- /dev/null +++ b/frontend/src/pages/lab/RunDetailPage.tsx @@ -0,0 +1,239 @@ +import { useState } from 'react' +import { useMutation, useQuery } from '@tanstack/react-query' +import { Link, useParams } from 'react-router-dom' +import { ArrowLeft, Download, FlaskConical, Loader2 } from 'lucide-react' +import { APP_CONTENT_MAX_WIDTH, PageContainer } from '../../components/layout/PageContainer' +import { StatusBadge } from '../../components/shared/StatusBadge' +import { useI18n } from '../../i18n/useI18n' +import { cn } from '../../lib/utils' +import { labApi, type LabRunDetail } from '../../api/lab' + +const CARD = 'rounded-xl border border-[#e5e7eb] bg-white shadow-[0_1px_3px_rgba(0,0,0,.04)]' +const TABLE_WRAPPER = 'overflow-x-auto rounded-xl border border-[#e5e7eb] bg-white shadow-[0_1px_3px_rgba(0,0,0,.04)]' +const SECONDARY_BUTTON = + 'inline-flex items-center gap-1.5 rounded-lg border border-slate-200 px-3 py-1.5 text-xs font-medium text-slate-600 transition-colors hover:bg-slate-50' +const TH = 'px-4 py-3 text-xs font-semibold uppercase tracking-wide text-[#9ca3af] whitespace-nowrap' +const RATE_METRICS = ['cer_micro', 'cer_macro', 'cer', 'der', 'jer', 'wer'] + +type T = ReturnType['t'] + +function mapStatusForBadge(status?: string): string { + const key = (status ?? 'pending').toLowerCase() + if (key === 'finished' || key === 'succeeded') return 'completed' + if (key === 'queued') return 'pending' + return key +} + +function fmtMetric(key: string | undefined, value: unknown): string { + if (typeof value !== 'number') return '—' + if (key && RATE_METRICS.includes(key)) return `${(value * 100).toFixed(2)}%` + if (key === 'rtf') return value.toFixed(2) + return value.toFixed(3) +} + +function fmtDuration(sec: unknown): string { + if (typeof sec !== 'number' || !sec) return '—' + if (sec < 60) return `${Math.round(sec)}s` + return `${Math.floor(sec / 60)}m ${Math.round(sec % 60)}s` +} + +function aggregateKey(scenario: string, arm?: string): string { + return !arm || arm === 'default' ? scenario : `${scenario}@${arm}` +} + +function MetaCard({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ) +} + +function ResultRow({ r, metricKey, t }: { r: Record; metricKey?: string; t: T }) { + const [open, setOpen] = useState(false) + const status = String(r.status ?? 'pending') + const err = r.error ? String(r.error) : '' + return ( + <> + + {String(r.sample_id ?? '—')} + {String(r.scenario ?? '—')} + + + {err ? ( + + ) : null} + + {fmtMetric(metricKey, r.primary_metric)} + {fmtDuration(r.duration_sec)} + {r.cached ? '✓' : ''} + + {err && open ? ( + + + 
+              {err}
+
+ + + ) : null} + + ) +} + +export function RunDetailPage() { + const { t } = useI18n() + const { runId = '' } = useParams() + const { data, isLoading, isError, refetch } = useQuery({ + queryKey: ['lab', 'run', runId], + queryFn: () => labApi.runDetail(runId), + retry: false, + enabled: Boolean(runId), + }) + const download = useMutation({ + mutationFn: async () => { + const md = await labApi.reportMarkdown(runId) + const blob = new Blob([md], { type: 'text/markdown;charset=utf-8' }) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `${runId}.md` + document.body.appendChild(a) + a.click() + a.remove() + URL.revokeObjectURL(url) + }, + }) + + const run = (data ?? {}) as LabRunDetail + const aggregates = (run.aggregates ?? {}) as Record> + const results = (run.results ?? []) as Array> + const samples = run.sample_count ?? run.num_samples + const started = run.started_at ?? run.created_at + const elapsed = run.elapsed_sec ?? run.duration_sec + + const metricKeyFor = (scenario: string, arm?: string): string | undefined => { + const agg = aggregates[aggregateKey(scenario, arm)] ?? aggregates[scenario] + return agg?.primary_metric as string | undefined + } + + return ( + + + + {t.lab.detail.back} + + + {isLoading ? ( +
{t.lab.loading}
+ ) : isError || !run.run_id ? ( +
+

{t.lab.detail.notFound}

+ +
+ ) : ( + <> +
+
+
+ +
+
+

{String(run.run_id)}

+

+ {t.lab.detail.suite}: {run.suite ?? '—'} + {' · '} + {t.lab.detail.dataset}: {run.dataset ?? '—'} +

+
+
+ +
+ +
+ + + + +
+ +
+

{t.lab.detail.aggregates}

+
+ + + + {[t.lab.detail.colScenario, t.lab.detail.colMetric, 'mean', 'micro', 'std', + t.lab.detail.colScored, t.lab.detail.colFailed, t.lab.detail.colSkipped].map(h => ( + + ))} + + + + {Object.entries(aggregates).length === 0 ? ( + + ) : Object.entries(aggregates).map(([key, agg]) => { + const metric = agg.primary_metric as string | undefined + const higher = agg.higher_is_better + const arrow = higher === false ? '↓' : higher ? '↑' : '' + const micro = (agg.corpus as Record | undefined)?.[`${metric}_micro`] + return ( + + + + + + + + + + + ) + })} + +
{h}
—
{key}{metric ?? '—'} {arrow}{fmtMetric(metric, agg.mean)}{fmtMetric(metric, micro)}{fmtMetric(metric, agg.std)}{String(agg.scored ?? 0)}{String(agg.failed ?? 0)}{String(agg.skipped ?? 0)}
+
+
+ +
+

{t.lab.detail.samplesTitle}

+
+ + + + {[t.lab.detail.colSample, t.lab.detail.colScenario, t.lab.detail.colStatus, + t.lab.detail.colMetric, t.lab.detail.colDuration, t.lab.detail.colCached].map(h => ( + + ))} + + + + {results.length === 0 ? ( + + ) : results.map((r, i) => ( + + ))} + +
{h}
{t.lab.detail.noSamples}
+
+
+ + )} + + ) +} + +export default RunDetailPage diff --git a/src/translip_lab/server/app.py b/src/translip_lab/server/app.py index 7afa57d..33ffabb 100644 --- a/src/translip_lab/server/app.py +++ b/src/translip_lab/server/app.py @@ -16,11 +16,12 @@ from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, PlainTextResponse from ..config import load_config from ..core.run_store import compare_runs, list_runs, load_run from ..core.scenario import SCENARIO_REGISTRY +from ..report.markdown import run_to_markdown from ..datasets import DATASET_REGISTRY, get_dataset from .. import scenarios as _scenarios # noqa: F401 — registers scenarios @@ -81,6 +82,19 @@ def api_run_detail(run_id: str) -> dict[str, Any]: return load_run(run_dir) +@app.get("/api/lab/runs/{run_id}/report.md") +def api_run_report(run_id: str) -> PlainTextResponse: + """Markdown report for a run (download).""" + run_dir = load_config().runs_dir / run_id + if not (run_dir / "run-manifest.json").is_file(): + raise HTTPException(status_code=404, detail=f"run not found: {run_id}") + md = run_to_markdown(load_run(run_dir)) + return PlainTextResponse( + md, media_type="text/markdown", + headers={"Content-Disposition": f'attachment; filename="{run_id}.md"'}, + ) + + @app.get("/api/lab/compare") def api_compare(baseline: str, candidate: str) -> dict[str, Any]: config = load_config() diff --git a/tests/lab/test_server.py b/tests/lab/test_server.py index ecac68e..69cf2a7 100644 --- a/tests/lab/test_server.py +++ b/tests/lab/test_server.py @@ -51,6 +51,23 @@ def test_runs_listing_and_detail(monkeypatch, tmp_path): assert client.get("/api/lab/runs/r1").json()["aggregates"]["asr"]["mean"] == 0.2 +def test_run_report_markdown(monkeypatch, tmp_path): + client = _client(monkeypatch, tmp_path) + assert client.get("/api/lab/runs/nope/report.md").status_code == 404 + run_dir = tmp_path / "runs" / "r1" + run_dir.mkdir(parents=True) + (run_dir / "run-manifest.json").write_text(json.dumps({ + "run_id": "r1", "suite": "s", "dataset": "d", "sample_count": 1, + "aggregates": {"asr": {"scenario": "asr", "primary_metric": "cer", "higher_is_better": False, "mean": 0.2}}, + "results": [{"sample_id": "x", "scenario": "asr", "status": "succeeded", "primary_metric": 0.2, "cached": False}], + }), encoding="utf-8") + r = client.get("/api/lab/runs/r1/report.md") + assert r.status_code == 200 + assert "text/markdown" in r.headers["content-type"] + assert "# Lab run `r1`" in r.text and "## Aggregates" in r.text + assert 'filename="r1.md"' in r.headers.get("content-disposition", "") + + def test_compare_endpoint(monkeypatch, tmp_path): client = _client(monkeypatch, tmp_path) for rid, mean in (("a", 0.2), ("b", 0.3)): From 6233ed6ed7c5b122ced42a12f76a5d2f85738be1 Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 11:57:10 +0800 Subject: [PATCH 09/10] =?UTF-8?q?feat(lab):=20real=20tracked=20run=20worke?= =?UTF-8?q?r=20=E2=80=94=20JobManager=20+=20status=20endpoints=20+=20UI=20?= =?UTF-8?q?polling=20(PRD=20P0-3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /api/lab/runs was fire-and-forget; it now submits a tracked job with a real lifecycle (queued -> running -> succeeded/failed), serialized through a single worker so heavy ML runs don't stampede a 16 GB host, with captured logs. Job id == run id, so a finished job links straight to its detail page. - server/jobs.py: JobManager (queue + single worker, log capture, in-memory + JSON records under /.jobs/ that survive a restart; orphaned running/queued jobs are marked failed on reload). - server/app.py: lazy per-runs-dir JobManager; POST /runs submits a job; GET /api/lab/jobs and GET /api/lab/jobs/{id} (+ log tail). - frontend: triggerRun returns job_id; ExperimentsTab shows a polling JobStatus that resolves to a "view results" link on success / "run failed"; mock job state machine for the offline demo; i18n (zh/en). - PRD 6.1: P0-1..P0-4 marked done. Tests: JobManager lifecycle incl. serial execution + log capture + orphan recovery, plus endpoints (tests/lab 115 passing). Browser-verified: mock job flow (run -> status -> results link -> detail), and a live server actually running the worker to a tracked terminal state with the subprocess output captured. --- ...2-lab-module-product-requirements.zh-CN.md | 10 +- frontend/src/api/lab.ts | 29 +++ frontend/src/api/labMock.ts | 52 +++++- frontend/src/i18n/messages.ts | 4 + frontend/src/pages/lab/LabPage.tsx | 72 +++++-- src/translip_lab/server/app.py | 62 +++++-- src/translip_lab/server/jobs.py | 175 ++++++++++++++++++ tests/lab/test_jobs.py | 92 +++++++++ tests/lab/test_server.py | 15 ++ 9 files changed, 474 insertions(+), 37 deletions(-) create mode 100644 src/translip_lab/server/jobs.py create mode 100644 tests/lab/test_jobs.py diff --git a/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md index dce5e94..d0062e4 100644 --- a/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md +++ b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md @@ -284,12 +284,12 @@ translip 是面向中文影视剧的字幕提取 / ASR / 翻译 / 配音流水 ### 6.1 P0（基础能力补齐） -| 编号 | 需求 | 价值 | +| 编号 | 需求 | 状态 / 落地 | |---|---|---| -| P0-1 | **Run 详情页**：点击 leaderboard 行进入，展示 manifest、前 N 条样本、stderr、所有指标 | 当前只能看汇总值，无法定位异常样本 | -| P0-2 | **失败可追溯**：failed run 的 stderr/log 在 UI 可见 | 调试效率 | -| P0-3 | **Lab server 真后端化**：当前触发是"拼装命令"，需让 server 真起 worker | 用户期望"点了就跑" | -| P0-4 | **Markdown 报告下载**：[report/markdown.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/report/markdown.py) 已有，前端 RegressionTab 接一个下载按钮即可 | 产出可分享物 | +| P0-1 ✅ | **Run 详情页**：点击 leaderboard 行进入，展示 manifest、样本指标、stderr | 已落地：`/lab/runs/:runId`（`RunDetailPage`）——头部 + 聚合指标表 + 样本结果表 | +| P0-2 ✅ | **失败可追溯**：failed 样本的 stderr/traceback 在 UI 可见 | 已落地：详情页每条失败样本「查看错误」可展开 stderr 面板 | +| P0-3 ✅ | **Lab server 真后端化**：tracked worker（queued→running→succeeded/failed），单 worker 串行 + 日志捕获 | 已落地：[server/jobs.py](file:///Users/yinyijun/OpenSourceProjects/translip/src/translip_lab/server/jobs.py) `JobManager` + `GET /api/lab/jobs[/{id}]` + 前端 `JobStatus` 轮询 | +| P0-4 ✅ | **Markdown 报告下载** | 已落地：`GET /api/lab/runs/{id}/report.md` + 详情页下载按钮 | ### 6.2 P1（产品力提升） diff --git a/frontend/src/api/lab.ts b/frontend/src/api/lab.ts index 19d7353..8dcdc39 100644 --- a/frontend/src/api/lab.ts +++ b/frontend/src/api/lab.ts @@ -1,11 +1,13 @@ import axios, { type AxiosError } from 'axios' import { MOCK_DATASETS, + MOCK_JOBS, MOCK_RUNS, MOCK_RUN_DETAILS, MOCK_SCENARIOS, MOCK_SUITES, buildMockCompare, + buildMockJob, buildMockReport, buildMockTriggerResponse, } from './labMock' @@ -140,6 +142,23 @@ export interface LabTriggerRunPayload { export interface LabTriggerRunResponse { status: string cmd: string[] + job_id?: string + run_id?: string +} + +export interface LabJob { + job_id: string + status: string // queued | running | succeeded | failed + suite?: string | null + dataset?: string | null + scenarios?: string[] + run_id?: string | null + returncode?: number | null + created_at?: string + started_at?: string | null + finished_at?: string | null + error?: string | null + log_tail?: string } export const labApi = { @@ -189,4 +208,14 @@ export const labApi = { () => labClient.post('/api/lab/runs', payload).then(r => r.data), () => buildMockTriggerResponse(payload), ), + jobs: () => + callOrFallback( + () => labClient.get('/api/lab/jobs').then(r => r.data), + () => MOCK_JOBS, + ), + jobDetail: (jobId: string) => + callOrFallback( + () => labClient.get(`/api/lab/jobs/${jobId}`).then(r => r.data), + () => buildMockJob(jobId), + ), } diff --git a/frontend/src/api/labMock.ts b/frontend/src/api/labMock.ts index f252ae8..3e410b4 100644 --- a/frontend/src/api/labMock.ts +++ b/frontend/src/api/labMock.ts @@ -1,6 +1,7 @@ import type { LabCompareResult, LabDataset, + LabJob, LabRunDetail, LabRunSummary, LabScenario, @@ -353,15 +354,62 @@ export function buildMockCompare(baseline: string, candidate: string): LabCompar } } +const _mockJobStart = new Map() + export function buildMockTriggerResponse(payload: LabTriggerRunPayload): LabTriggerRunResponse { - const cmd = ['translip-lab', 'run'] + const label = payload.suite ?? payload.dataset ?? 'run' + const jobId = `mock-${label}-${Date.now()}` + const cmd = ['translip-lab', 'run', '--run-id', jobId] if (payload.suite) cmd.push('--suite', payload.suite) if (payload.dataset) cmd.push('--dataset', payload.dataset) if (payload.limit) cmd.push('--limit', String(payload.limit)) if (payload.no_cache) cmd.push('--no-cache') - return { status: 'started', cmd } + _mockJobStart.set(jobId, Date.now()) + return { status: 'queued', cmd, job_id: jobId, run_id: jobId } +} + +// A tiny client-side state machine so the offline demo shows queued → running → +// succeeded without a backend (the real flow polls GET /api/lab/jobs/{id}). +export function buildMockJob(jobId: string): LabJob { + if (!_mockJobStart.has(jobId)) _mockJobStart.set(jobId, Date.now()) + const t0 = _mockJobStart.get(jobId) ?? Date.now() + const elapsed = Date.now() - t0 + const status = elapsed < 1600 ? 'running' : 'succeeded' + const suite = jobId.replace(/^mock-/, '').replace(/-\d+$/, '') + const matchRun = MOCK_RUNS.find(r => r.suite === suite) + return { + job_id: jobId, + status, + suite, + run_id: status === 'succeeded' ? (matchRun?.run_id ?? jobId) : null, + created_at: new Date(t0).toISOString(), + started_at: new Date(t0).toISOString(), + finished_at: status === 'succeeded' ? new Date().toISOString() : null, + log_tail: + status === 'succeeded' + ? '$ translip-lab run …\n[3/3] sample · scenario → succeeded\n\nreport: …/report.html\n' + : '$ translip-lab run …\n[1/3] sample · scenario → running…\n', + } } +export const MOCK_JOBS: LabJob[] = [ + { + job_id: 'mock-tts-clone-synthetic-001', + status: 'running', + suite: 'tts-clone-synthetic', + run_id: null, + created_at: '2026-06-23 10:12:00', + }, + { + job_id: 'mock-asr-drama-wenetspeech-002', + status: 'succeeded', + suite: 'asr-drama-wenetspeech', + run_id: '20260621-1842-asr-drama-paraformer', + created_at: '2026-06-23 09:00:00', + finished_at: '2026-06-23 09:05:26', + }, +] + export function buildMockReport(runId: string): string { const r = MOCK_RUN_DETAILS[runId] ?? MOCK_RUNS.find(x => x.run_id === runId) if (!r) return `# Lab run \`${runId}\`\n\n(not found — demo data)\n` diff --git a/frontend/src/i18n/messages.ts b/frontend/src/i18n/messages.ts index a83e270..710055f 100644 --- a/frontend/src/i18n/messages.ts +++ b/frontend/src/i18n/messages.ts @@ -2195,6 +2195,8 @@ const zhMessages = { tagErase: '字幕擦除', tagDub: '配音端到端', tagClone: '音色克隆', + viewResults: '查看结果', + jobFailed: '运行失败', }, leaderboard: { heading: '历史 Run', @@ -4453,6 +4455,8 @@ const enMessages: LocaleMessages = { tagErase: 'Erase', tagDub: 'End-to-end Dub', tagClone: 'Voice Clone', + viewResults: 'View results', + jobFailed: 'Run failed', }, leaderboard: { heading: 'Past runs', diff --git a/frontend/src/pages/lab/LabPage.tsx b/frontend/src/pages/lab/LabPage.tsx index fad97c9..f809048 100644 --- a/frontend/src/pages/lab/LabPage.tsx +++ b/frontend/src/pages/lab/LabPage.tsx @@ -22,6 +22,7 @@ import { labApi, type LabCompareResult, type LabDataset, + type LabJob, type LabRunSummary, type LabSource, } from '../../api/lab' @@ -46,7 +47,7 @@ const TABLE_WRAPPER = function mapStatusForBadge(status?: string): string { const key = (status ?? 'pending').toLowerCase() - if (key === 'finished') return 'completed' + if (key === 'finished' || key === 'succeeded') return 'completed' if (key === 'queued') return 'pending' return key } @@ -273,6 +274,36 @@ function DatasetsTab() { ) } +function JobStatus({ jobId }: { jobId: string }) { + const { t } = useI18n() + const { data } = useQuery({ + queryKey: ['lab', 'job', jobId], + queryFn: () => labApi.jobDetail(jobId), + retry: false, + refetchInterval: query => { + const s = (query.state.data as LabJob | undefined)?.status + return s === 'succeeded' || s === 'failed' ? false : 1200 + }, + }) + const status = data?.status ?? 'queued' + return ( +
+ + {status === 'succeeded' && data?.run_id ? ( + + {t.lab.experiments.viewResults} + + + ) : status === 'failed' ? ( + {t.lab.experiments.jobFailed} + ) : null} +
+ ) +} + function ExperimentsTab() { const { t } = useI18n() const suitesQuery = useQuery({ @@ -281,8 +312,12 @@ function ExperimentsTab() { staleTime: 60_000, retry: false, }) + const [jobs, setJobs] = useState>({}) const triggerMutation = useMutation({ mutationFn: (suite: string) => labApi.triggerRun({ suite, limit: 3 }), + onSuccess: (data, suite) => { + if (data.job_id) setJobs(prev => ({ ...prev, [suite]: data.job_id! })) + }, }) if (suitesQuery.isLoading) return
@@ -322,7 +357,6 @@ function ExperimentsTab() {

{t.lab.experiments.hint}

{suites.map(suite => { - const started = triggerMutation.isSuccess && triggerMutation.variables === suite const tags = inferTags(suite) const dataset = inferDataset(suite) return ( @@ -338,21 +372,25 @@ function ExperimentsTab() {
-

{t.lab.experiments.smokeRun}

- + {jobs[suite] ? ( + + ) : ( + <> +

{t.lab.experiments.smokeRun}

+ + + )}
) diff --git a/src/translip_lab/server/app.py b/src/translip_lab/server/app.py index 33ffabb..f6c904b 100644 --- a/src/translip_lab/server/app.py +++ b/src/translip_lab/server/app.py @@ -8,7 +8,6 @@ from __future__ import annotations import os -import subprocess import sys import threading from pathlib import Path @@ -24,6 +23,7 @@ from ..report.markdown import run_to_markdown from ..datasets import DATASET_REGISTRY, get_dataset from .. import scenarios as _scenarios # noqa: F401 — registers scenarios +from .jobs import JobManager _WEB_DIR = Path(__file__).parent / "web" _SUITES_DIR = Path(__file__).resolve().parent.parent / "suites" @@ -34,6 +34,22 @@ allow_methods=["*"], allow_headers=["*"], ) +# One JobManager per runs_dir, created lazily — so tests with isolated +# TRANSLIP_LAB_HOME don't share a worker/queue, and a deployed server keeps one. +_job_managers: dict[str, JobManager] = {} +_jm_lock = threading.Lock() + + +def get_job_manager() -> JobManager: + runs_dir = load_config().runs_dir + key = str(runs_dir) + with _jm_lock: + jm = _job_managers.get(key) + if jm is None: + jm = JobManager(runs_dir=runs_dir) + _job_managers[key] = jm + return jm + @app.get("/", response_class=HTMLResponse) def index() -> str: @@ -106,23 +122,43 @@ def api_compare(baseline: str, candidate: str) -> dict[str, Any]: @app.post("/api/lab/runs") def api_trigger_run(payload: dict[str, Any]) -> dict[str, Any]: - """Launch a run in the background (subprocess → translip-lab run).""" - cmd = [sys.executable, "-m", "translip_lab", "run"] - if payload.get("suite"): - cmd += ["--suite", str(payload["suite"])] - elif payload.get("dataset") and payload.get("scenarios"): - scenarios = payload["scenarios"] - scenarios = ",".join(scenarios) if isinstance(scenarios, list) else str(scenarios) - cmd += ["--dataset", str(payload["dataset"]), "--scenario", scenarios] + """Submit a tracked run job (queued → running → succeeded/failed; serialized).""" + suite = payload.get("suite") + dataset = payload.get("dataset") + raw_scenarios = payload.get("scenarios") + tail: list[str] = [] + if suite: + tail += ["--suite", str(suite)] + elif dataset and raw_scenarios: + joined = ",".join(raw_scenarios) if isinstance(raw_scenarios, list) else str(raw_scenarios) + tail += ["--dataset", str(dataset), "--scenario", joined] else: raise HTTPException(status_code=400, detail="provide 'suite' or ('dataset' and 'scenarios')") if payload.get("limit") is not None: - cmd += ["--limit", str(payload["limit"])] + tail += ["--limit", str(payload["limit"])] if payload.get("no_cache"): - cmd += ["--no-cache"] + tail += ["--no-cache"] + + scenarios = raw_scenarios if isinstance(raw_scenarios, list) else ([raw_scenarios] if raw_scenarios else []) + jm = get_job_manager() + job_id = jm.new_job_id(suite or dataset) + cmd = [sys.executable, "-m", "translip_lab", "run", "--run-id", job_id, *tail] + job = jm.submit(cmd=cmd, job_id=job_id, suite=suite, dataset=dataset, scenarios=scenarios) + return {"status": job.status, "job_id": job.job_id, "run_id": job.job_id, "cmd": job.cmd} + + +@app.get("/api/lab/jobs") +def api_jobs() -> list[dict[str, Any]]: + return get_job_manager().list_jobs() + - threading.Thread(target=lambda: subprocess.run(cmd, cwd=os.getcwd()), daemon=True).start() - return {"status": "started", "cmd": cmd} +@app.get("/api/lab/jobs/{job_id}") +def api_job_detail(job_id: str) -> dict[str, Any]: + jm = get_job_manager() + job = jm.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"job not found: {job_id}") + return {**job.to_dict(), "log_tail": jm.tail_log(job_id)} def run_server(host: str | None = None, port: int | None = None) -> None: diff --git a/src/translip_lab/server/jobs.py b/src/translip_lab/server/jobs.py new file mode 100644 index 0000000..f9ef511 --- /dev/null +++ b/src/translip_lab/server/jobs.py @@ -0,0 +1,175 @@ +"""Background job runner for the lab dashboard. + +Turns "trigger a run" from a fire-and-forget subprocess into a *tracked* job with a +lifecycle (``queued`` → ``running`` → ``succeeded`` | ``failed``). Jobs are serialized +through a single worker thread so heavy ML runs don't stampede a 16 GB host, their +output is captured to a log, and a small JSON record per job is persisted under +``/.jobs/`` so the list survives a server restart and is inspectable. + +The job id is reused as the run id (``--run-id``), so a finished job points straight +at ``//run-manifest.json`` (the detail page / report endpoint). The +worker shells out to the same ``python -m translip_lab run`` the CLI uses — one-way +coupling, models freed on subprocess exit, exactly like every other lab stage. +""" +from __future__ import annotations + +import json +import subprocess +import threading +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path +from queue import Queue +from typing import Any + +_TERMINAL = ("succeeded", "failed") + + +def _safe(text: str) -> str: + return "".join(c if (c.isalnum() or c in "-_.") else "_" for c in text) + + +@dataclass +class Job: + job_id: str + cmd: list[str] + suite: str | None = None + dataset: str | None = None + scenarios: list[str] = field(default_factory=list) + status: str = "queued" # queued | running | succeeded | failed + run_id: str | None = None # == job_id once a run-manifest exists + returncode: int | None = None + created_at: str = "" + started_at: str | None = None + finished_at: str | None = None + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class JobManager: + def __init__(self, *, runs_dir: Path, max_workers: int = 1) -> None: + self.runs_dir = Path(runs_dir) + self.jobs_dir = self.runs_dir / ".jobs" + self.jobs_dir.mkdir(parents=True, exist_ok=True) + self._jobs: dict[str, Job] = {} + self._lock = threading.Lock() + self._seq = 0 + self._queue: Queue[str] = Queue() + self._load_existing() + for _ in range(max(1, max_workers)): + threading.Thread(target=self._worker, daemon=True).start() + + # ---- ids / paths ------------------------------------------------------- + def _now(self) -> str: + return datetime.now().astimezone().isoformat(timespec="seconds") + + def new_job_id(self, label: str | None) -> str: + with self._lock: + self._seq += 1 + seq = self._seq + stamp = datetime.now().strftime("%Y%m%dT%H%M%S") + return f"{stamp}-{_safe(label or 'run')}-{seq:03d}" + + def log_path(self, job_id: str) -> Path: + return self.jobs_dir / f"{job_id}.log" + + def _record_path(self, job_id: str) -> Path: + return self.jobs_dir / f"{job_id}.json" + + # ---- persistence ------------------------------------------------------- + def _persist(self, job: Job) -> None: + try: + self._record_path(job.job_id).write_text( + json.dumps(job.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8") + except OSError: + pass + + def _load_existing(self) -> None: + for rec in self.jobs_dir.glob("*.json"): + try: + data = json.loads(rec.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + fields = {k: data.get(k) for k in Job.__dataclass_fields__ if k in data} + try: + job = Job(**fields) + except TypeError: + continue + # a job still 'queued'/'running' from a prior process is orphaned + if job.status not in _TERMINAL: + job.status = "failed" + job.error = job.error or "interrupted (server restarted)" + self._persist(job) + self._jobs[job.job_id] = job + + # ---- api --------------------------------------------------------------- + def submit(self, *, cmd: list[str], job_id: str, suite: str | None = None, + dataset: str | None = None, scenarios: list[str] | None = None) -> Job: + job = Job(job_id=job_id, cmd=list(cmd), suite=suite, dataset=dataset, + scenarios=list(scenarios or []), created_at=self._now()) + with self._lock: + self._jobs[job_id] = job + self._persist(job) + self._queue.put(job_id) + return job + + def list_jobs(self) -> list[dict[str, Any]]: + with self._lock: + jobs = sorted(self._jobs.values(), key=lambda j: j.created_at, reverse=True) + return [j.to_dict() for j in jobs] + + def get_job(self, job_id: str) -> Job | None: + return self._jobs.get(job_id) + + def tail_log(self, job_id: str, n: int = 8000) -> str: + path = self.log_path(job_id) + if not path.is_file(): + return "" + return path.read_text(encoding="utf-8", errors="replace")[-n:] + + # ---- worker ------------------------------------------------------------ + def _worker(self) -> None: + while True: + job_id = self._queue.get() + try: + self._run_job(job_id) + except Exception: # noqa: BLE001 — never let the worker thread die + job = self._jobs.get(job_id) + if job and job.status not in _TERMINAL: + job.status = "failed" + job.finished_at = self._now() + self._persist(job) + finally: + self._queue.task_done() + + def _run_job(self, job_id: str) -> None: + job = self._jobs.get(job_id) + if job is None: + return + job.status = "running" + job.started_at = self._now() + self._persist(job) + try: + with self.log_path(job_id).open("w", encoding="utf-8") as fh: + fh.write(f"$ {' '.join(job.cmd)}\n\n") + fh.flush() + proc = subprocess.run(job.cmd, stdout=fh, stderr=subprocess.STDOUT) + job.returncode = proc.returncode + except Exception as exc: # noqa: BLE001 — surface as a failed job + job.status = "failed" + job.error = f"{type(exc).__name__}: {exc}" + job.finished_at = self._now() + self._persist(job) + return + # A produced run-manifest means the run actually completed (a non-zero exit + # may just be a --fail-under gate, which still yields real results). + if (self.runs_dir / job_id / "run-manifest.json").is_file(): + job.status = "succeeded" + job.run_id = job_id + else: + job.status = "failed" + job.error = f"exit {job.returncode}; no run-manifest produced — see log" + job.finished_at = self._now() + self._persist(job) diff --git a/tests/lab/test_jobs.py b/tests/lab/test_jobs.py new file mode 100644 index 0000000..c8f1778 --- /dev/null +++ b/tests/lab/test_jobs.py @@ -0,0 +1,92 @@ +"""JobManager lifecycle — real subprocesses (python -c), no ML, fully offline.""" +from __future__ import annotations + +import json +import sys +import time + +from translip_lab.server.jobs import Job, JobManager + + +def _wait(jm: JobManager, job_id: str, timeout: float = 15.0) -> Job: + deadline = time.time() + timeout + while time.time() < deadline: + job = jm.get_job(job_id) + if job and job.status in ("succeeded", "failed"): + return job + time.sleep(0.02) + raise AssertionError(f"job {job_id} did not finish in {timeout}s") + + +def _writes_manifest_cmd(run_dir) -> list[str]: + code = ( + "import pathlib,sys;" + "d=pathlib.Path(sys.argv[1]);d.mkdir(parents=True,exist_ok=True);" + "(d/'run-manifest.json').write_text('{}')" + ) + return [sys.executable, "-c", code, str(run_dir)] + + +def test_job_succeeds_when_manifest_produced(tmp_path): + jm = JobManager(runs_dir=tmp_path / "runs") + jm.submit(cmd=_writes_manifest_cmd(tmp_path / "runs" / "t1"), job_id="t1", suite="s") + job = _wait(jm, "t1") + assert job.status == "succeeded" + assert job.run_id == "t1" and job.returncode == 0 + assert job.started_at and job.finished_at + + +def test_job_fails_without_manifest(tmp_path): + jm = JobManager(runs_dir=tmp_path / "runs") + jm.submit(cmd=[sys.executable, "-c", "import sys; sys.exit(2)"], job_id="t2") + job = _wait(jm, "t2") + assert job.status == "failed" and job.returncode == 2 and job.run_id is None + assert "no run-manifest" in (job.error or "") + + +def test_log_is_captured(tmp_path): + jm = JobManager(runs_dir=tmp_path / "runs") + jm.submit(cmd=[sys.executable, "-c", "print('HELLO-LAB-LOG')"], job_id="t3") + _wait(jm, "t3") + assert "HELLO-LAB-LOG" in jm.tail_log("t3") + + +def test_jobs_run_serially(tmp_path): + jm = JobManager(runs_dir=tmp_path / "runs") + marks = tmp_path / "marks" + marks.mkdir() + code = ( + "import time,pathlib,sys;" + "p=pathlib.Path(sys.argv[1]);" + "p.write_text(str(time.time()));" + "time.sleep(0.4);" + "p.write_text(p.read_text()+' '+str(time.time()))" + ) + jm.submit(cmd=[sys.executable, "-c", code, str(marks / "a")], job_id="a") + jm.submit(cmd=[sys.executable, "-c", code, str(marks / "b")], job_id="b") + _wait(jm, "a") + _wait(jm, "b") + a_start, a_end = (float(x) for x in (marks / "a").read_text().split()) + b_start, b_end = (float(x) for x in (marks / "b").read_text().split()) + first_end, second_start = (a_end, b_start) if a_start < b_start else (b_end, a_start) + assert first_end <= second_start + 0.1 # single worker → active windows don't overlap + + +def test_list_and_get(tmp_path): + jm = JobManager(runs_dir=tmp_path / "runs") + jm.submit(cmd=[sys.executable, "-c", "pass"], job_id="z1", suite="s1") + _wait(jm, "z1") + assert any(j["job_id"] == "z1" for j in jm.list_jobs()) + assert jm.get_job("z1").suite == "s1" + assert jm.get_job("missing") is None + + +def test_orphan_jobs_marked_failed_on_reload(tmp_path): + runs = tmp_path / "runs" + (runs / ".jobs").mkdir(parents=True) + (runs / ".jobs" / "stale.json").write_text(json.dumps({ + "job_id": "stale", "cmd": [], "status": "running", "created_at": "2026-01-01T00:00:00", + }), encoding="utf-8") + jm = JobManager(runs_dir=runs) # reload picks up the orphan + stale = jm.get_job("stale") + assert stale is not None and stale.status == "failed" and "interrupted" in (stale.error or "") diff --git a/tests/lab/test_server.py b/tests/lab/test_server.py index 69cf2a7..c731ac2 100644 --- a/tests/lab/test_server.py +++ b/tests/lab/test_server.py @@ -68,6 +68,21 @@ def test_run_report_markdown(monkeypatch, tmp_path): assert 'filename="r1.md"' in r.headers.get("content-disposition", "") +def test_trigger_job_and_jobs_endpoints(monkeypatch, tmp_path): + client = _client(monkeypatch, tmp_path) + assert client.post("/api/lab/runs", json={}).status_code == 400 # missing args + + body = client.post("/api/lab/runs", json={"suite": "asr-diar-ramc", "limit": 1}).json() + assert body["job_id"] and body["run_id"] == body["job_id"] + assert "--run-id" in body["cmd"] and "--suite" in body["cmd"] # job id == run id + + jobs = client.get("/api/lab/jobs").json() + assert any(j["job_id"] == body["job_id"] for j in jobs) + detail = client.get(f"/api/lab/jobs/{body['job_id']}") + assert detail.status_code == 200 and "log_tail" in detail.json() + assert client.get("/api/lab/jobs/does-not-exist").status_code == 404 + + def test_compare_endpoint(monkeypatch, tmp_path): client = _client(monkeypatch, tmp_path) for rid, mean in (("a", 0.2), ("b", 0.3)): From 50177d234c534954a54835cc013a6cc6368389fb Mon Sep 17 00:00:00 2001 From: Sherlock Yin Date: Tue, 23 Jun 2026 14:39:20 +0800 Subject: [PATCH 10/10] feat(lab): leaderboard scenario/status/search filters (PRD P1-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The leaderboard mixed all scenarios, making single-scenario decisions hard. Add a filter row above the table: - scenario filter — when set, runs are filtered to that scenario AND ranked/shown by that scenario's metric (handles scenario and scenario@arm aggregate keys), not just the first metric found across the run - status filter (finished/running/failed) + a run-id/suite search box, with a " / " count Pure frontend over existing run data; i18n (zh/en). Time-window filter deferred. Verified: build + 189 vitest + eslint clean; browser (system Chrome) — scenario, status, and search filters each narrow to the right rows (0 page errors). --- ...2-lab-module-product-requirements.zh-CN.md | 2 +- frontend/src/i18n/messages.ts | 16 ++++ frontend/src/pages/lab/LabPage.tsx | 90 +++++++++++++++++-- 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md index d0062e4..07fbe93 100644 --- a/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md +++ b/docs/superpowers/specs/2026-06-22-lab-module-product-requirements.zh-CN.md @@ -295,7 +295,7 @@ translip 是面向中文影视剧的字幕提取 / ASR / 翻译 / 配音流水 | 编号 | 需求 | 价值 | |---|---|---| -| P1-1 | **Leaderboard 按场景过滤** + 按时间窗口过滤 | 当前混排不利于做单场景决策 | +| P1-1 🟡 | **Leaderboard 过滤**：场景 / 状态 / 搜索（已落地）+ 时间窗口（待补） | 已落地：选场景即按该场景主指标排名，解决「混排不利于单场景决策」 | | P1-2 | **多 run 横向对比**（>2 个） | 选模型时一次看 4-5 个候选 | | P1-3 | **数据集一键下载向导**（含 WenetSpeech EULA 引导） | 降低使用门槛 | | P1-4 | **样本级 hypothesis vs reference diff** | 定位 CER 高来自哪些 sample | diff --git a/frontend/src/i18n/messages.ts b/frontend/src/i18n/messages.ts index 710055f..1c7bda2 100644 --- a/frontend/src/i18n/messages.ts +++ b/frontend/src/i18n/messages.ts @@ -2202,6 +2202,14 @@ const zhMessages = { heading: '历史 Run', hint: '按主指标（数值越小越好的字段，如 CER/DER/MCD；其他指标越大越好）排序。', empty: '还没有任何 run，先在「评测实验」里发起一次。', + filterScenario: '按场景过滤', + filterStatus: '按状态过滤', + allScenarios: '全部场景', + allStatuses: '全部状态', + statusFinished: '已完成', + statusRunning: '运行中', + statusFailed: '失败', + searchPlaceholder: '搜索 run / 套件', columns: { rank: '#', runId: 'Run ID', @@ -4462,6 +4470,14 @@ const enMessages: LocaleMessages = { heading: 'Past runs', hint: 'Sorted by primary metric (CER/DER/MCD lower-is-better; other metrics higher-is-better).', empty: 'No runs yet — launch one from the Experiments tab first.', + filterScenario: 'Filter by scenario', + filterStatus: 'Filter by status', + allScenarios: 'All scenarios', + allStatuses: 'All statuses', + statusFinished: 'Finished', + statusRunning: 'Running', + statusFailed: 'Failed', + searchPlaceholder: 'Search run / suite', columns: { rank: '#', runId: 'Run ID', diff --git a/frontend/src/pages/lab/LabPage.tsx b/frontend/src/pages/lab/LabPage.tsx index f809048..7df56d5 100644 --- a/frontend/src/pages/lab/LabPage.tsx +++ b/frontend/src/pages/lab/LabPage.tsx @@ -7,10 +7,12 @@ import { CheckCircle2, Database, ExternalLink, + Filter, FlaskConical, Loader2, Minus, PlayCircle, + Search, Sparkles, Trophy, } from 'lucide-react' @@ -45,6 +47,11 @@ const CARD = const TABLE_WRAPPER = 'overflow-x-auto rounded-xl border border-[#e5e7eb] bg-white shadow-[0_1px_3px_rgba(0,0,0,.04)]' +const SELECT = + 'rounded-lg border border-[#e5e7eb] bg-white px-2.5 py-1.5 text-xs text-[#374151] transition-colors hover:border-[#d1d5db] focus:border-[#3b5bdb] focus:outline-none focus:ring-1 focus:ring-[#3b5bdb]' + +const METRIC_KEYS = ['sim', 'cer_micro', 'cer', 'der', 'mcd', 'si_sdr', 'f1', 'psnr'] + function mapStatusForBadge(status?: string): string { const key = (status ?? 'pending').toLowerCase() if (key === 'finished' || key === 'succeeded') return 'completed' @@ -136,7 +143,7 @@ function StatCard({ icon: Icon, label, value, hint }: { icon: typeof Database; l function pickPrimaryMetric(run: LabRunSummary): { scenario: string; key: string; value: number } | null { const aggregates = run.aggregates ?? {} for (const [scenario, metrics] of Object.entries(aggregates)) { - for (const k of ['sim', 'cer_micro', 'cer', 'der', 'mcd', 'si_sdr', 'f1', 'psnr']) { + for (const k of METRIC_KEYS) { const v = metrics[k] if (typeof v === 'number') return { scenario, key: k, value: v } } @@ -144,6 +151,21 @@ function pickPrimaryMetric(run: LabRunSummary): { scenario: string; key: string; return null } +// When a scenario filter is active, rank/show that scenario's metric (handles +// "scenario" and "scenario@arm" aggregate keys), not just the first metric found. +function pickMetricForScenario(run: LabRunSummary, scenario: string): { scenario: string; key: string; value: number } | null { + if (scenario === 'all') return pickPrimaryMetric(run) + for (const [aggKey, metrics] of Object.entries(run.aggregates ?? {})) { + if (aggKey === scenario || aggKey.startsWith(`${scenario}@`)) { + for (const k of METRIC_KEYS) { + const v = metrics[k] + if (typeof v === 'number') return { scenario, key: k, value: v } + } + } + } + return null +} + function formatPercent(value: number): string { return `${(value * 100).toFixed(2)}%` } @@ -410,21 +432,75 @@ function LeaderboardTab() { }) const runs = useMemo(() => data ?? [], [data]) + const [scenarioFilter, setScenarioFilter] = useState('all') + const [statusFilter, setStatusFilter] = useState('all') + const [search, setSearch] = useState('') + + const allScenarios = useMemo(() => { + const set = new Set() + runs.forEach(r => (r.scenarios ?? []).forEach(s => set.add(s))) + return Array.from(set).sort() + }, [runs]) + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase() + return runs.filter(r => { + if (scenarioFilter !== 'all' && !(r.scenarios ?? []).includes(scenarioFilter)) return false + if (statusFilter !== 'all' && (r.status ?? '') !== statusFilter) return false + if (q && !`${r.run_id} ${r.suite ?? ''}`.toLowerCase().includes(q)) return false + return true + }) + }, [runs, scenarioFilter, statusFilter, search]) + + const pick = (run: LabRunSummary) => pickMetricForScenario(run, scenarioFilter) const ranked = useMemo(() => { - return [...runs].sort((a, b) => { - const pa = pickPrimaryMetric(a) - const pb = pickPrimaryMetric(b) + return [...filtered].sort((a, b) => { + const pa = pickMetricForScenario(a, scenarioFilter) + const pb = pickMetricForScenario(b, scenarioFilter) if (!pa) return 1 if (!pb) return -1 const lower = LOWER_IS_BETTER.has(pa.key) return lower ? pa.value - pb.value : pb.value - pa.value }) - }, [runs]) - const bestRunId = ranked.find(r => r.status === 'finished' && pickPrimaryMetric(r))?.run_id + }, [filtered, scenarioFilter]) + const bestRunId = ranked.find(r => r.status === 'finished' && pick(r))?.run_id return (

{t.lab.leaderboard.hint}

+
+ + + +
+ + setSearch(e.target.value)} + placeholder={t.lab.leaderboard.searchPlaceholder} + className="rounded-lg border border-[#e5e7eb] bg-white py-1.5 pl-7 pr-2.5 text-xs text-[#374151] focus:border-[#3b5bdb] focus:outline-none focus:ring-1 focus:ring-[#3b5bdb]" + /> +
+ {ranked.length} / {runs.length} +
{isLoading ? ( @@ -450,7 +526,7 @@ function LeaderboardTab() { {ranked.map((run, idx) => { - const primary = pickPrimaryMetric(run) + const primary = pick(run) const rtf = run.aggregates?.asr?.rtf const isBest = run.run_id === bestRunId const isRunning = run.status === 'running'