test(mutation): Introduce mutation-based test coverage

Author: talagrand

.github/workflows/mutants.yml

@@ -0,0 +1,139 @@
+name: Mutation Testing
+
+on:
+  pull_request:
+    paths:
+      - ".cargo/*.toml"
+      - ".github/workflows/mutants.yml"
+      - "Cargo.*"
+      - "src/**"
+      - "tests/**"
+
+env:
+  CARGO_TERM_COLOR: always
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  mutants:
+    name: Mutation Testing
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2
+
+      - name: Install cargo-mutants
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-mutants
+
+      - name: Relative diff
+        run: |
+          git branch -av
+          git diff origin/${{ github.base_ref }}.. | tee git.diff
+
+      - name: Run cargo-mutants
+        run: cargo mutants --no-shuffle --all-features --in-diff git.diff --in-place
+        continue-on-error: true
+
+      - name: Check mutants report
+        id: report
+        run: |
+          if [ -f mutants.out/outcomes.json ]; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Comment on PR
+        if: steps.report.outputs.exists == 'true' && (github.event.pull_request.head.repo.full_name == github.repository)
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            const report = JSON.parse(fs.readFileSync('mutants.out/outcomes.json', 'utf8'));
+
+            const counts = {
+              caught: Number(report.caught ?? 0),
+              missed: Number(report.missed ?? 0),
+              timeout: Number(report.timeout ?? 0),
+              unviable: Number(report.unviable ?? 0),
+            };
+
+            const missed = (report.outcomes ?? [])
+              .filter(o => o.summary === 'MissedMutant' && o.scenario?.Mutant)
+              .map(o => {
+                const mutant = o.scenario.Mutant;
+                const functionName = mutant.function?.function_name ?? mutant.name;
+                const replacement = mutant.replacement ?? 'unknown replacement';
+                return `${functionName}: ${replacement}`;
+              });
+
+            const totalMutants = Number(report.total_mutants ?? 0);
+            const testable = counts.caught + counts.missed;
+            const score = testable > 0 ? (counts.caught / testable * 100).toFixed(1) : 'N/A';
+
+            let body = '## 🧬 Mutation Testing Results\n\n';
+            body += 'Scope: PR diff (`--in-diff`)\n\n';
+            body += '| Metric | Count |\n|--------|-------|\n';
+            body += `| Caught | ${counts.caught} |\n`;
+            body += `| Missed | ${counts.missed} |\n`;
+            body += `| Timeout | ${counts.timeout} |\n`;
+            body += `| Unviable | ${counts.unviable} |\n`;
+            body += `| **Mutation Score** | **${score}%** |\n\n`;
+
+            if (totalMutants === 0) {
+              body += 'No mutants were generated for the PR diff. This is expected for test-only or non-Rust changes.\n';
+            } else if (missed.length > 0) {
+              body += '<details><summary>Missed mutants (test gaps)</summary>\n\n```\n';
+              body += missed.join('\n');
+              body += '\n```\n\n</details>\n';
+            } else {
+              body += 'All mutants in the PR diff were caught by tests.\n';
+            }
+
+            const marker = '## 🧬 Mutation Testing Results';
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const existing = comments.find(c => c.body?.includes(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+            }
+
+      - name: No report notice
+        if: steps.report.outputs.exists == 'false'
+        run: echo "cargo-mutants did not produce mutants.out/outcomes.json; skipping PR comment and artifact upload."
+
+      - name: Upload mutants report
+        if: steps.report.outputs.exists == 'true'
+        uses: actions/upload-artifact@v7
+        with:
+          name: mutants-report
+          path: mutants.out/
+          retention-days: 14

.gitignore

@@ -1,4 +1,7 @@
 # Generated by Cargo
 target/
 
+# Mutation testing output
+mutants.out/
+
 .vscode/

src/ast.rs

@@ -501,6 +501,30 @@ mod helper_function_tests {
 
         assert!(builtin_fn != func); // cross-variant catch-all arm
 
+        // PartialEq: Functions with different params are not equal
+        let func_different_params =
+            eval(&parse_scheme("(lambda (y) (+ y 1))").unwrap(), &mut env).unwrap();
+        assert!(func != func_different_params);
+
+        // PartialEq: Functions with same params/body but different captured environments are not equal
+        let mut env3 = create_global_env();
+        eval(&parse_scheme("(define z 99)").unwrap(), &mut env3).unwrap();
+        let func_different_env =
+            eval(&parse_scheme("(lambda (x) (+ x 1))").unwrap(), &mut env3).unwrap();
+        assert!(func != func_different_env);
+
+        // PartialEq: PrecompiledOps with same op but different args are not equal
+        let precompiled_different_args = parse_scheme("(+ 3 4)").unwrap();
+        assert!(precompiled != precompiled_different_args);
+
+        // PartialEq: Unspecified never equals non-Unspecified values
+        assert!(Value::Unspecified != val(0));
+        assert!(Value::Unspecified != val(false));
+        assert!(Value::Unspecified != val(""));
+        assert!(Value::Unspecified != nil());
+        assert!(val(0) != Value::Unspecified);
+        assert!(val(false) != Value::Unspecified);
+
         // Error Display: (error, expected_substring)
         let error_cases: Vec<(crate::Error, &str)> = vec![
             (

src/builtinops.rs

@@ -674,6 +674,10 @@ mod tests {
                 && (add_ref.op_kind != if_ref.op_kind)
                 && (*add_ref == add_ref.clone())
         );
+
+        // BuiltinOp::eq: different builtins are not equal
+        let sub_ref = find_scheme_op("-").unwrap();
+        assert!(add_ref != sub_ref);
     }
 
     /// Macro to create test cases, invoking builtins via the registry.

src/evaluator.rs

@@ -1089,11 +1089,18 @@ mod tests {
             ("(+ 1 (car \"not-a-list\"))", Error),
             ("(if (not 42) 1 2)", Error),
             // Test lambda parameter errors
-            ("(lambda (x x) x)", Error),           // Duplicate params
-            ("(lambda \"not-a-list\" 42)", Error), // Invalid params
+            ("(lambda (x x) x)", Error), // Duplicate params
+            (
+                "(lambda \"not-a-list\" 42)",
+                SpecificError("must be a list"),
+            ), // Invalid params
+            ("(lambda 42 x)", SpecificError("must be a list")), // Non-list params
             // Test define errors
-            ("(define 123 42)", Error),            // Invalid var name
-            ("(define \"not-symbol\" 42)", Error), // Invalid var name
+            ("(define 123 42)", SpecificError("define requires a symbol")),
+            (
+                "(define \"not-symbol\" 42)",
+                SpecificError("define requires a symbol"),
+            ),
             // === ERROR CASES ===
             // Unbound variables
             (

src/jsonlogic.rs

@@ -512,6 +512,8 @@ mod tests {
                 r#"{"123invalid": [1, 2]}"#,
                 SpecificError("Invalid operator name"),
             ),
+            // Invalid var names should be rejected
+            (r#"{"var": "123"}"#, SpecificError("valid symbol")),
             // Design validation tests - operations intentionally rejected/different
             (
                 r#"{"unknown_not": [true]}"#,
@@ -803,10 +805,38 @@ mod tests {
                 "{label} should fail conversion"
             );
         }
+    }
 
-        // Non-symbol list converts to JSON array
-        let list_val = Value::List(vec![Value::Number(1), Value::Number(2)]);
-        assert_eq!(ast_to_jsonlogic(&list_val).unwrap(), "[1,2]");
+    #[test]
+    fn test_ast_to_jsonlogic_roundtrip() {
+        // AST values that should convert to specific JSON representations
+        let roundtrip_cases: Vec<(Value, &str)> = vec![
+            // Non-symbol list converts to JSON array
+            (
+                Value::List(vec![Value::Number(1), Value::Number(2)]),
+                "[1,2]",
+            ),
+            // Empty list converts to empty JSON array
+            (Value::List(vec![]), "[]"),
+            // PrecompiledOp "list" roundtrips to JSON array
+            (parse_scheme("(list 1 2 3)").unwrap(), "[1,2,3]"),
+            // Unprecompiled Value::List with Symbol("list") also roundtrips to JSON array
+            (
+                Value::List(vec![
+                    Value::Symbol("list".into()),
+                    Value::Number(4),
+                    Value::Number(5),
+                ]),
+                "[4,5]",
+            ),
+        ];
+        for (value, expected_json) in &roundtrip_cases {
+            assert_eq!(
+                ast_to_jsonlogic(value).unwrap(),
+                *expected_json,
+                "Roundtrip failed for {value:?}"
+            );
+        }
     }
 
     /// Helper function to test AST equivalence and roundtrip (shared by Identical and IdenticalWithEvalError)

src/scheme.rs

@@ -695,6 +695,8 @@ mod tests {
             ("(not #f)", precompiled_op("not", vec![val(false)])),
             // Test nested arity errors are also caught
             ("(list (not) 42)", SpecificError("ArityError")),
+            // Arity error nested inside unprecompiled list (unknown-fn isn't a builtin, so outer remains Value::List)
+            ("(unknown-fn (if #t 1))", SpecificError("ArityError")),
             // ===== PARSE-TIME SYNTAX ERRORS =====
             // Unclosed parentheses - should contain parse error information
             ("(+ 1 (- 2", SpecificError("ParseError")),
@@ -737,6 +739,12 @@ mod tests {
             ")".repeat(MAX_PARSE_DEPTH)
         );
         let deep_quotes_at_limit = format!("{}a", "'".repeat(MAX_PARSE_DEPTH));
+        // Longhand (quote ...) nesting exercises the depth tracking in parse_list's quote path
+        let deep_longhand_quote_at_limit = format!(
+            "{}a{}",
+            "(quote ".repeat(MAX_PARSE_DEPTH),
+            ")".repeat(MAX_PARSE_DEPTH)
+        );
 
         let depth_test_cases = vec![
             // At/over limit should fail at parse time with specific error
@@ -748,6 +756,11 @@ mod tests {
                 deep_quotes_at_limit.as_str(),
                 SpecificError("Invalid syntax"),
             ),
+            // Longhand (quote ...) nesting should also be caught
+            (
+                deep_longhand_quote_at_limit.as_str(),
+                SpecificError("Invalid syntax"),
+            ),
         ];
 
         run_parse_tests(depth_test_cases);