fix(desktop): harden evaluation evidence edges

Author: xr843

desktop/src/app.rs

@@ -99,6 +99,7 @@ impl EvaluationWindow {
 fn suite_matches_filter(
     row: &SkillRow,
     latest_result: Option<&EvaluationRunResult>,
+    latest_graded_result: Option<&EvaluationRunResult>,
     filter: SuiteFilter,
 ) -> bool {
     match filter {
@@ -107,7 +108,7 @@ fn suite_matches_filter(
         SuiteFilter::Attention => row.quality_level() == QualityLevel::Attention,
         SuiteFilter::Missing => row.quality_level() == QualityLevel::Missing,
         SuiteFilter::NotRun => latest_result.is_none(),
-        SuiteFilter::FailedRun => latest_result.is_some_and(|result| {
+        SuiteFilter::FailedRun => latest_graded_result.is_some_and(|result| {
             !result.is_dry_run()
                 && result.total_count > 0
                 && result.passed_count < result.total_count
@@ -1521,6 +1522,12 @@ impl MasterSkillApp {
             .into_iter()
             .map(|result| (result.slug.clone(), result))
             .collect();
+        let latest_graded_results: BTreeMap<_, _> = self
+            .traces
+            .latest_graded_evaluation_results_by_slug()
+            .into_iter()
+            .map(|result| (result.slug.clone(), result))
+            .collect();
         ui.horizontal_wrapped(|ui| {
             ui.label("Filter");
             for filter in [
@@ -1543,8 +1550,12 @@ impl MasterSkillApp {
             .clone()
             .into_iter()
             .filter(|row| {
-                suite_matches_filter(row, latest_results.get(&row.slug), self.suite_filter)
-                    && suite_matches_query(row, &self.suite_query)
+                suite_matches_filter(
+                    row,
+                    latest_results.get(&row.slug),
+                    latest_graded_results.get(&row.slug),
+                    self.suite_filter,
+                ) && suite_matches_query(row, &self.suite_query)
             })
             .collect();
         if rows.is_empty() {
@@ -2507,35 +2518,48 @@ mod tests {
             mode: EvaluationMode::Graded,
             trace_id: 2,
         };
+        let later_dry_run = EvaluationRunResult {
+            slug: "zhiyi".to_string(),
+            passed_count: 0,
+            total_count: 10,
+            mode: EvaluationMode::DryRun,
+            trace_id: 3,
+        };
 
         assert!(super::suite_matches_filter(
             &ready,
             Some(&passing_run),
+            Some(&passing_run),
             super::SuiteFilter::Ready
         ));
         assert!(super::suite_matches_filter(
             &attention,
+            Some(&later_dry_run),
             Some(&failed_run),
             super::SuiteFilter::Attention
         ));
         assert!(super::suite_matches_filter(
             &missing,
             None,
+            None,
             super::SuiteFilter::Missing
         ));
         assert!(super::suite_matches_filter(
             &missing,
             None,
+            None,
             super::SuiteFilter::NotRun
         ));
         assert!(super::suite_matches_filter(
             &attention,
+            Some(&later_dry_run),
             Some(&failed_run),
             super::SuiteFilter::FailedRun
         ));
         assert!(!super::suite_matches_filter(
             &ready,
             Some(&passing_run),
+            Some(&passing_run),
             super::SuiteFilter::FailedRun
         ));
     }

desktop/src/trace.rs

@@ -226,9 +226,11 @@ impl EvaluationRunHistoryItem {
     }
 
     fn pass_rate_basis(&self) -> usize {
-        (self.passed_count * 10_000)
-            .checked_div(self.total_count)
-            .unwrap_or(0)
+        if self.total_count == 0 {
+            return 0;
+        }
+        let basis = (self.passed_count as u128 * 10_000) / self.total_count as u128;
+        basis.min(usize::MAX as u128) as usize
     }
 
     pub fn matches_filter(&self, filter: EvaluationRunHistoryFilter) -> bool {
@@ -1071,11 +1073,19 @@ fn evaluation_case_statuses(
 ) -> Result<Vec<EvaluationCaseStatus>, String> {
     cases
         .iter()
-        .map(|case| match EvaluationCaseStatus::from_wire(&case.status) {
-            EvaluationCaseStatus::Unknown(status) => Err(format!(
-                "invalid fidelity suite: unknown case status {status:?}"
-            )),
-            status => Ok(status),
+        .map(|case| {
+            if case.index.checked_add(1).is_none() {
+                return Err(format!(
+                    "invalid fidelity suite: case index {} cannot be displayed one-based",
+                    case.index
+                ));
+            }
+            match EvaluationCaseStatus::from_wire(&case.status) {
+                EvaluationCaseStatus::Unknown(status) => Err(format!(
+                    "invalid fidelity suite: unknown case status {status:?}"
+                )),
+                status => Ok(status),
+            }
         })
         .collect()
 }
@@ -1272,6 +1282,14 @@ fn compare_evaluation_runs(
     }
 }
 
+fn signed_usize_delta(current: usize, previous: usize) -> isize {
+    if current >= previous {
+        current.saturating_sub(previous).min(isize::MAX as usize) as isize
+    } else {
+        -(previous.saturating_sub(current).min(isize::MAX as usize) as isize)
+    }
+}
+
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct TraceSummary {
     pub total: usize,
@@ -2153,10 +2171,10 @@ impl TraceStore {
                 previous_trace_id: previous.trace_id,
                 current_failed_count,
                 previous_failed_count,
-                failed_delta: current_failed_count as isize - previous_failed_count as isize,
+                failed_delta: signed_usize_delta(current_failed_count, previous_failed_count),
                 current_pass_rate,
                 previous_pass_rate,
-                pass_rate_delta_points: current_pass_rate as isize - previous_pass_rate as isize,
+                pass_rate_delta_points: signed_usize_delta(current_pass_rate, previous_pass_rate),
                 action: current.action.clone(),
             });
         }
@@ -3209,6 +3227,47 @@ mod tests {
         assert!(errors[0].message.contains("SKIPPED"));
     }
 
+    #[test]
+    fn rejects_case_index_that_cannot_be_displayed_one_based() {
+        let mut store = TraceStore::new(10);
+        let run = store.begin_with_action(
+            "Running master-huineng fidelity",
+            TraceAction::FidelityDryRunSkill {
+                slug: "huineng".to_string(),
+            },
+            Some("python3 scripts/test-fidelity.py --master master-huineng --json"),
+            "Queued.",
+        );
+        store.finish_success_with_detail(
+            run,
+            "master-huineng fidelity finished",
+            format!(
+                r#"[{{
+                  "schema_version": 1,
+                  "master": "master-huineng",
+                  "mode": "graded",
+                  "outcome": "completed",
+                  "total": 1,
+                  "passed": 1,
+                  "failed": 0,
+                  "results": [
+                    {{"index": {}, "question": "overflow", "status": "PASS"}}
+                  ]
+                }}]"#,
+                usize::MAX
+            ),
+            Duration::from_millis(50),
+        );
+
+        let errors = store.records.back().unwrap().evaluation_errors();
+        assert_eq!(errors.len(), 1);
+        assert_eq!(
+            errors[0].kind,
+            EvaluationEvidenceErrorKind::MalformedPayload
+        );
+        assert!(errors[0].message.contains("index"));
+    }
+
     #[test]
     fn rejects_evaluation_payload_for_wrong_skill_scope() {
         let mut store = TraceStore::new(10);
@@ -4139,6 +4198,24 @@ mod tests {
         );
     }
 
+    #[test]
+    fn pass_rate_calculation_handles_large_legacy_counts() {
+        let item = EvaluationRunHistoryItem {
+            trace_id: 1,
+            scope: "master-huineng".to_string(),
+            status: TraceStatus::Succeeded,
+            passed_count: usize::MAX,
+            total_count: usize::MAX,
+            failed_count: 0,
+            mode: EvaluationMode::Graded,
+            duration_ms: None,
+            action: None,
+            trend: EvaluationRunTrend::New,
+        };
+
+        assert_eq!(item.pass_rate_percent(), 100);
+    }
+
     #[test]
     fn annotates_evaluation_run_history_with_scope_trends() {
         let mut store = TraceStore::new(10);
@@ -4808,6 +4885,47 @@ mod tests {
         );
     }
 
+    #[test]
+    fn saturates_large_legacy_regression_deltas() {
+        let mut store = TraceStore::new(10);
+        let old_run = store.begin_with_action(
+            "Running master-huineng fidelity",
+            TraceAction::FidelityDryRunSkill {
+                slug: "huineng".to_string(),
+            },
+            Some("legacy fidelity"),
+            "Queued.",
+        );
+        store.finish_success_with_detail(
+            old_run,
+            "legacy fidelity finished",
+            "Testing: master-huineng\nResult: 0/1 passed (0%)",
+            Duration::from_millis(50),
+        );
+        let new_run = store.begin_with_action(
+            "Running master-huineng fidelity",
+            TraceAction::FidelityDryRunSkill {
+                slug: "huineng".to_string(),
+            },
+            Some("legacy fidelity"),
+            "Queued.",
+        );
+        store.finish_success_with_detail(
+            new_run,
+            "legacy fidelity finished",
+            format!(
+                "Testing: master-huineng\nResult: 0/{} passed (0%)",
+                usize::MAX
+            ),
+            Duration::from_millis(50),
+        );
+
+        let regressions = store.evaluation_regressions(8);
+
+        assert_eq!(regressions.len(), 1);
+        assert_eq!(regressions[0].failed_delta, isize::MAX);
+    }
+
     #[test]
     fn persists_trace_history_and_next_record_id() {
         let path = temp_path("trace-history");

docs/superpowers/plans/2026-07-30-desktop-evaluation-contract.md

@@ -755,4 +755,3 @@ Record:
 - exact verification commands and outcomes;
 - backward-compatibility behavior;
 - intentionally deferred PR 2 runtime work.
-

scripts/test-fidelity.py

@@ -91,11 +91,22 @@ def load_tests(master_dir: Path) -> list[dict]:
     ):
         if line.strip():
             try:
-                tests.append(json.loads(line))
+                test = json.loads(line)
             except json.JSONDecodeError as error:
                 raise ValueError(
                     f"Invalid fidelity.jsonl line {line_number}: {error.msg}"
                 ) from error
+            if not isinstance(test, dict):
+                raise ValueError(
+                    f"Invalid fidelity.jsonl line {line_number}: expected a JSON object"
+                )
+            question = test.get("q")
+            if not isinstance(question, str) or not question.strip():
+                raise ValueError(
+                    f"Invalid fidelity.jsonl line {line_number}: "
+                    "expected a non-empty string q"
+                )
+            tests.append(test)
     return tests
 
 

tests/test_fidelity_exit.py

@@ -97,6 +97,26 @@ def test_invalid_fidelity_file_emits_versioned_error_suite(runner, monkeypatch,
     assert "line 2" in suite["error"]
 
 
+def test_fidelity_case_without_question_emits_versioned_error_suite(
+    runner, monkeypatch, tmp_path
+):
+    master_dir = tmp_path / "master-broken"
+    (master_dir / "tests").mkdir(parents=True)
+    (master_dir / "tests" / "fidelity.jsonl").write_text(
+        '{"difficulty":"basic"}\n',
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(runner, "PREBUILT_DIR", tmp_path)
+
+    suite = runner.run_tests("master-broken", dry_run=True, quiet=True)
+
+    assert suite["outcome"] == "error"
+    assert suite["total"] == 0
+    assert suite["results"] == []
+    assert "line 1" in suite["error"]
+    assert "non-empty string q" in suite["error"]
+
+
 def test_missing_master_exits_nonzero_with_clean_json_stdout():
     result = subprocess.run(
         [