From e19b2636930497291aac59b2bd3703b0fc2f6fc5 Mon Sep 17 00:00:00 2001 From: WellDunDun <45949032+WellDunDun@users.noreply.github.com> Date: Sat, 14 Mar 2026 16:49:52 +0300 Subject: [PATCH 1/4] feat: implement 8 HumanLayer-informed audit improvements with compare command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply extracted wisdom from HumanLayer's harness engineering analysis to expand the Reins scoring model and CLI capabilities. Back-pressure and hooks promoted from findings to scored points (max 22→24), AGENTS.md uses tiered evaluation, MCP tool proliferation detected, language-agnostic back-pressure covers JS/Python/ Rust/Make, plugin system via .reins/custom-checks.json, and new compare command for tracking audit progression over time. All docs updated to reflect new thresholds and scoring dimensions. Co-Authored-By: Claude Opus 4.6 --- README.md | 43 +++--- cli/reins/README.md | 30 ++-- cli/reins/package.json | 3 +- cli/reins/src/index.test.ts | 173 +++++++++++++++++++++- cli/reins/src/index.ts | 22 ++- cli/reins/src/lib/audit/context.ts | 135 ++++++++++++++++- cli/reins/src/lib/audit/scoring.ts | 93 +++++++++++- cli/reins/src/lib/commands/compare.ts | 86 +++++++++++ cli/reins/src/lib/commands/evolve.test.ts | 8 +- cli/reins/src/lib/commands/evolve.ts | 16 +- cli/reins/src/lib/types.ts | 2 +- docs/design-docs/core-beliefs.md | 4 + docs/design-docs/symphony-integration.md | 2 +- package.json | 3 +- skill/Reins/HarnessMethodology.md | 26 +++- skill/Reins/SKILL.md | 17 ++- skill/Reins/Workflows/Audit.md | 26 ++-- 17 files changed, 609 insertions(+), 80 deletions(-) create mode 100644 cli/reins/src/lib/commands/compare.ts diff --git a/README.md b/README.md index 160bafe..94a955c 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ npx reins-cli@latest audit . ```json { "total_score": 6, - "max_score": 22, + "max_score": 24, "maturity_level": "L1: Inloop", "recommendations": [ "Create ARCHITECTURE.md with domain map and layer rules", @@ -95,7 +95,7 @@ Most agent rollouts fail for one boring reason: agents can edit code, but the re Reins gives you a repeatable operating system for agent work: - **A map** (`AGENTS.md`, architecture docs, indexed decisions) -- **A score** (0-22 maturity audit with concrete gaps) +- **A score** (0-24 maturity audit with concrete gaps) - **A plan** (next-step evolution path by maturity level) - **A guardrail model** (`risk-policy.json` + CI enforcement signals) @@ -138,28 +138,31 @@ block-beta **Use them together.** Reins scaffolds your repo so `AGENTS.md` tells the agent where everything is, `ARCHITECTURE.md` defines the rules, and golden principles are enforced in CI. Then a session orchestrator runs the actual coding work on top of that well-structured repo. -## The four commands +## The five commands ```mermaid graph LR - Init["reins init\nScaffold"] --> Audit["reins audit\nScore 0-22"] + Init["reins init\nScaffold"] --> Audit["reins audit\nScore 0-24"] Audit --> Evolve["reins evolve\nLevel up"] Evolve --> Doctor["reins doctor\nHealth check"] Doctor --> Audit + Audit --> Compare["reins compare\nBaseline diff"] style Init fill:#1e3a5f,stroke:#3b82f6,color:#e2e8f0 style Audit fill:#1e3a5f,stroke:#60a5fa,color:#e2e8f0 style Evolve fill:#1e3a5f,stroke:#818cf8,color:#e2e8f0 style Doctor fill:#1e3a5f,stroke:#a78bfa,color:#e2e8f0 + style Compare fill:#1e3a5f,stroke:#c084fc,color:#e2e8f0 ``` ```bash reins init . # Scaffold the full structure reins init . --pack auto # Adaptive pack selection from project signals reins init . --pack agent-factory # Optional advanced automation pack -reins audit . # Score against harness principles (0-22) +reins audit . # Score against harness principles (0-24) reins evolve . # Roadmap to next maturity level reins doctor . # Health check with prescriptive fixes +reins compare # Compare current audit against a baseline ``` ## The maturity model @@ -168,10 +171,10 @@ Every repo sits on a maturity spectrum. The audit tells you where you are. The e ```mermaid graph LR - L0["L0: Manual\n0-5"] --> L1["L1: Inloop\n6-11"] - L1 --> L2["L2: Guided Outloop\n12-16"] - L2 --> L3["L3: Full Outloop\n17-19"] - L3 --> L4["L4: Zero Touch\n20-22"] + L0["L0: Manual\n0-6"] --> L1["L1: Inloop\n7-12"] + L1 --> L2["L2: Guided Outloop\n13-18"] + L2 --> L3["L3: Full Outloop\n19-21"] + L3 --> L4["L4: Zero Touch\n22-24"] style L0 fill:#1e293b,stroke:#475569,color:#94a3b8 style L1 fill:#1e293b,stroke:#3b82f6,color:#93c5fd @@ -182,11 +185,11 @@ graph LR | Score | Level | What it means | |-------|-------|---------------| -| 0-5 | **L0: Manual** | Traditional engineering, no agent infra | -| 6-11 | **L1: Inloop** | Agents help, humans still write code | -| 12-16 | **L2: Guided Outloop** | Humans steer, agents execute most code | -| 17-19 | **L3: Full Outloop** | Agents handle full lifecycle | -| 20-22 | **L4: Zero Touch** | System maintains and improves itself | +| 0-6 | **L0: Manual** | Traditional engineering, no agent infra | +| 7-12 | **L1: Inloop** | Agents help, humans still write code | +| 13-18 | **L2: Guided Outloop** | Humans steer, agents execute most code | +| 19-21 | **L3: Full Outloop** | Agents handle full lifecycle | +| 22-24 | **L4: Zero Touch** | System maintains and improves itself | ## What `reins init` scaffolds @@ -232,16 +235,16 @@ reins init . --pack agent-factory ## The six audit dimensions -Scored across six dimensions, totaling 0-22: +Scored across six dimensions, totaling 0-24: ```mermaid graph TD - Score["Total Score\n0-22"] + Score["Total Score\n0-24"] RK["Repository Knowledge\n0-4"] AE["Architecture Enforcement\n0-3"] - AL["Agent Legibility\n0-4"] + AL["Agent Legibility\n0-5"] GP["Golden Principles\n0-3"] - AW["Agent Workflow\n0-5"] + AW["Agent Workflow\n0-6"] GC["Garbage Collection\n0-3"] RK --> Score @@ -269,14 +272,14 @@ graph TD | **Agent Workflow** | Agent config, risk policy, PR templates, CI enforcement, orchestration readiness | | **Garbage Collection** | Debt tracking, doc-gardening, quality grades, docs drift rules | -## Self-audit: 19/22 +## Self-audit: 19/24 Reins audits itself in CI. Current score: ```json { "total_score": 19, - "max_score": 22, + "max_score": 24, "maturity_level": "L3: Full Outloop" } ``` diff --git a/cli/reins/README.md b/cli/reins/README.md index 43cb718..11ccfc0 100644 --- a/cli/reins/README.md +++ b/cli/reins/README.md @@ -67,26 +67,26 @@ Pack modes: ### `reins audit ` -Score a project against harness engineering principles (0-22): +Score a project against harness engineering principles (0-24): ```bash reins audit . ``` -Scores six dimensions (variable max per dimension, 22 total): +Scores six dimensions (variable max per dimension, 24 total): 1. **Repository Knowledge** (0-4) — AGENTS.md, docs/, versioned plans 2. **Architecture Enforcement** (0-3) — ARCHITECTURE.md, dependency rules, linters -3. **Agent Legibility** (0-4) — Bootable app, observability, lean dependencies +3. **Agent Legibility** (0-5) — Bootable app, observability, lean dependencies 4. **Golden Principles** (0-3) — Documented rules, CI enforcement, cleanup process -5. **Agent Workflow** (0-5) — Agent config, PR templates, merge gates, orchestration readiness +5. **Agent Workflow** (0-6) — Agent config, PR templates, merge gates, orchestration readiness 6. **Garbage Collection** (0-3) — Debt tracking, doc-gardening, quality grades Returns a maturity level: -- **L0: Manual** (0-5) — Traditional engineering -- **L1: Inloop** (6-11) — Agents help, humans still code -- **L2: Guided Outloop** (12-16) — Humans steer, agents execute -- **L3: Full Outloop** (17-19) — Agents handle full lifecycle -- **L4: Zero Touch** (20-22) — System maintains itself +- **L0: Manual** (0-6) — Traditional engineering +- **L1: Inloop** (7-12) — Agents help, humans still code +- **L2: Guided Outloop** (13-18) — Humans steer, agents execute +- **L3: Full Outloop** (19-21) — Agents handle full lifecycle +- **L4: Zero Touch** (22-24) — System maintains itself ### `reins evolve ` @@ -109,6 +109,18 @@ reins doctor . Returns pass/fail/warn for each check with specific fix instructions. +### `reins compare ` + +Compare the current audit score against a saved baseline: + +```bash +reins audit . > baseline.json +# ... make changes ... +reins compare . baseline.json +``` + +Returns a dimension-by-dimension diff showing score deltas and maturity level changes, useful for tracking progress over time or validating that a set of changes improved readiness. + ## Output All commands output deterministic JSON, making them composable with other tools: diff --git a/cli/reins/package.json b/cli/reins/package.json index 58a100e..751bc7e 100644 --- a/cli/reins/package.json +++ b/cli/reins/package.json @@ -36,7 +36,8 @@ ], "scripts": { "start": "bun src/index.ts", - "test": "bun test" + "test": "bun test", + "typecheck": "tsc --noEmit" }, "devDependencies": { "@types/bun": "latest" diff --git a/cli/reins/src/index.test.ts b/cli/reins/src/index.test.ts index b0ae518..9d74764 100644 --- a/cli/reins/src/index.test.ts +++ b/cli/reins/src/index.test.ts @@ -183,7 +183,7 @@ describe("reins audit", () => { const result = JSON.parse(stdout); expect(result.total_score).toBe(0); - expect(result.max_score).toBe(22); + expect(result.max_score).toBe(24); expect(result.maturity_level).toBe("L0: Manual"); expect(result.scores.repository_knowledge.score).toBe(0); expect(result.scores.architecture_enforcement.score).toBe(0); @@ -268,7 +268,7 @@ describe("reins audit", () => { expect(result).toHaveProperty("max_score"); expect(result).toHaveProperty("maturity_level"); expect(result).toHaveProperty("recommendations"); - expect(result.max_score).toBe(22); + expect(result.max_score).toBe(24); }); test("fails on nonexistent directory", async () => { @@ -1056,3 +1056,172 @@ describe("docs contract — skill/cli/human model clarity", () => { expect(cliReadme).toContain("npx skills add WellDunDun/reins"); }); }); + +// ─── Compare Command ───────────────────────────────────────────────────────── + +describe("reins compare", () => { + test("outputs score deltas against a baseline", async () => { + const dir = tmpDir("compare-basic"); + await runCli(`init ${dir}`); + + // Save a baseline + const { stdout: baselineJson } = await runCli(`audit ${dir}`); + const baselinePath = join(dir, "baseline.json"); + writeFileSync(baselinePath, baselineJson); + + // Add linter to change score + writeFileSync(join(dir, "biome.json"), "{}"); + + const { stdout, exitCode } = await runCli(`compare ${dir} ${baselinePath}`); + expect(exitCode).toBe(0); + + const result = JSON.parse(stdout); + expect(result.command).toBe("compare"); + expect(result.score_delta).toBeGreaterThanOrEqual(0); + expect(result.dimensions).toBeInstanceOf(Array); + expect(result.dimensions.length).toBe(6); + expect(result.dimensions[0]).toHaveProperty("dimension"); + expect(result.dimensions[0]).toHaveProperty("delta"); + }); + + test("shows maturity level change", async () => { + const dir = tmpDir("compare-level"); + + // Baseline with empty dir (L0) + const { stdout: baselineJson } = await runCli(`audit ${dir}`); + const baselinePath = join(dir, "baseline.json"); + writeFileSync(baselinePath, baselineJson); + + // Scaffold to raise score + await runCli(`init ${dir} --force`); + + const { stdout } = await runCli(`compare ${dir} ${baselinePath}`); + const result = JSON.parse(stdout); + + expect(result.before_level).toBe("L0: Manual"); + expect(result.level_changed).toBe(true); + }); + + test("fails with missing baseline", async () => { + const dir = tmpDir("compare-missing"); + const { stderr, exitCode } = await runCli(`compare ${dir} ${dir}/nonexistent.json`); + expect(exitCode).toBe(1); + expect(stderr).toContain("Baseline file not found"); + }); + + test("fails without baseline argument", async () => { + const dir = tmpDir("compare-no-arg"); + const { stderr, exitCode } = await runCli(`compare ${dir}`); + expect(exitCode).toBe(1); + expect(stderr).toContain("Missing baseline file"); + }); +}); + +// ─── MCP Tool Proliferation ───────────────────────────────────────────────── + +describe("MCP tool proliferation detection", () => { + test("warns when more than 10 MCP servers configured", async () => { + const dir = tmpDir("mcp-proliferation"); + mkdirSync(join(dir, ".claude"), { recursive: true }); + const servers: Record = {}; + for (let i = 0; i < 12; i++) { + servers[`tool-${i}`] = { command: "echo" }; + } + writeFileSync(join(dir, ".claude", "mcp.json"), JSON.stringify({ mcpServers: servers })); + + const { stdout } = await runCli(`audit ${dir}`); + const result = JSON.parse(stdout); + + expect(result.scores.agent_workflow.findings).toEqual( + expect.arrayContaining([expect.stringContaining("proliferation warning: 12 servers")]), + ); + }); + + test("reports healthy tool count when under 10", async () => { + const dir = tmpDir("mcp-healthy"); + mkdirSync(join(dir, ".claude"), { recursive: true }); + writeFileSync( + join(dir, ".claude", "mcp.json"), + JSON.stringify({ mcpServers: { tool1: {}, tool2: {}, tool3: {} } }), + ); + + const { stdout } = await runCli(`audit ${dir}`); + const result = JSON.parse(stdout); + + expect(result.scores.agent_workflow.findings).toEqual( + expect.arrayContaining([expect.stringContaining("MCP tools configured: 3 servers (healthy)")]), + ); + }); +}); + +// ─── Custom Checks Plugin ─────────────────────────────────────────────────── + +describe("custom checks plugin", () => { + test("file-exists check appears in audit findings", async () => { + const dir = tmpDir("custom-file-exists"); + mkdirSync(join(dir, ".reins"), { recursive: true }); + writeFileSync(join(dir, "SECURITY.md"), "# Security policy"); + writeFileSync( + join(dir, ".reins", "custom-checks.json"), + JSON.stringify({ + checks: [ + { name: "Security policy", type: "file-exists", path: "SECURITY.md", dimension: "repository_knowledge" }, + ], + }), + ); + + const { stdout } = await runCli(`audit ${dir}`); + const result = JSON.parse(stdout); + + expect(result.scores.repository_knowledge.findings).toEqual( + expect.arrayContaining([expect.stringContaining("[custom] Security policy: SECURITY.md exists")]), + ); + }); + + test("file-exists check reports missing file", async () => { + const dir = tmpDir("custom-file-missing"); + mkdirSync(join(dir, ".reins"), { recursive: true }); + writeFileSync( + join(dir, ".reins", "custom-checks.json"), + JSON.stringify({ + checks: [ + { name: "Missing file", type: "file-exists", path: "NOPE.md", dimension: "repository_knowledge" }, + ], + }), + ); + + const { stdout } = await runCli(`audit ${dir}`); + const result = JSON.parse(stdout); + + expect(result.scores.repository_knowledge.findings).toEqual( + expect.arrayContaining([expect.stringContaining("[custom] Missing file: NOPE.md missing")]), + ); + }); + + test("file-contains check matches pattern", async () => { + const dir = tmpDir("custom-contains"); + mkdirSync(join(dir, ".reins"), { recursive: true }); + writeFileSync(join(dir, "README.md"), "# My Project\n\nThis project uses TypeScript."); + writeFileSync( + join(dir, ".reins", "custom-checks.json"), + JSON.stringify({ + checks: [ + { + name: "TypeScript mentioned", + type: "file-contains", + path: "README.md", + pattern: "TypeScript", + dimension: "agent_legibility", + }, + ], + }), + ); + + const { stdout } = await runCli(`audit ${dir}`); + const result = JSON.parse(stdout); + + expect(result.scores.agent_legibility.findings).toEqual( + expect.arrayContaining([expect.stringContaining("[custom] TypeScript mentioned: pattern found")]), + ); + }); +}); diff --git a/cli/reins/src/index.ts b/cli/reins/src/index.ts index 526cc11..fee2c06 100644 --- a/cli/reins/src/index.ts +++ b/cli/reins/src/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun import { runAudit, runAuditCommand } from "./lib/commands/audit"; +import { runCompare } from "./lib/commands/compare"; import { runDoctor } from "./lib/commands/doctor"; import { runEvolve } from "./lib/commands/evolve"; import { runInit as runInitCommand } from "./lib/commands/init"; @@ -16,6 +17,7 @@ COMMANDS: audit Audit a project against harness engineering principles evolve Show evolution path to next maturity level doctor Check project health with prescriptive fixes + compare Compare current audit against a saved baseline help Show this help message OPTIONS: @@ -36,11 +38,11 @@ EXAMPLES: reins doctor . # Get prescriptive fixes MATURITY LEVELS: - L0: Manual (0-5) Traditional engineering - L1: Inloop (6-10) Agents in the loop, humans code - L2: Guided Outloop (11-15) Humans steer, agents execute - L3: Full Outloop (16-18) Agents handle full lifecycle - L4: Zero Touch (19-21) System maintains itself + L0: Manual (0-6) Traditional engineering + L1: Inloop (7-12) Agents in the loop, humans code + L2: Guided Outloop (13-18) Humans steer, agents execute + L3: Full Outloop (19-21) Agents handle full lifecycle + L4: Zero Touch (22-24) System maintains itself `; console.log(help); } @@ -81,6 +83,16 @@ function main(): void { runEvolve(path, hasFlag("--apply"), { runAudit, runInit: runInitCommand }); break; } + case "compare": { + const path = args[1] || "."; + const baseline = args[2]; + if (!baseline) { + console.error(JSON.stringify({ error: "Missing baseline file. Usage: reins compare " })); + process.exit(1); + } + runCompare(path, baseline, { runAudit }); + break; + } case "doctor": { const path = args[1] || "."; runDoctor(path); diff --git a/cli/reins/src/lib/audit/context.ts b/cli/reins/src/lib/audit/context.ts index c638d78..160db02 100644 --- a/cli/reins/src/lib/audit/context.ts +++ b/cli/reins/src/lib/audit/context.ts @@ -40,9 +40,21 @@ export interface AuditRuntimeContext { hasConcurrencyLimits: boolean; hasMergeProtection: boolean; hasSpecDocument: boolean; + hasHooksConfig: boolean; + hasBackPressure: boolean; + mcpToolCount: number; + customChecks: CustomCheck[]; frameworksDetected: string[]; } +export interface CustomCheck { + name: string; + type: "file-exists" | "file-contains"; + path: string; + pattern?: string; + dimension: string; +} + export function createAuditResult(projectName: string): AuditResult { return { project: projectName, @@ -51,13 +63,13 @@ export function createAuditResult(projectName: string): AuditResult { scores: { repository_knowledge: { score: 0, max: 4, findings: [] }, architecture_enforcement: { score: 0, max: 3, findings: [] }, - agent_legibility: { score: 0, max: 4, findings: [] }, + agent_legibility: { score: 0, max: 5, findings: [] }, golden_principles: { score: 0, max: 3, findings: [] }, - agent_workflow: { score: 0, max: 5, findings: [] }, + agent_workflow: { score: 0, max: 6, findings: [] }, garbage_collection: { score: 0, max: 3, findings: [] }, }, total_score: 0, - max_score: 22, + max_score: 24, maturity_level: "L0", recommendations: [], frameworks_detected: [], @@ -113,6 +125,115 @@ function detectConcurrencyLimits(targetDir: string, hasWorkflowConfig: boolean, } } +function detectHooksConfig(targetDir: string): boolean { + const hookPaths = [ + join(targetDir, ".claude", "settings.json"), + join(targetDir, ".husky"), + join(targetDir, ".githooks"), + join(targetDir, ".lefthook.yml"), + join(targetDir, "lefthook.yml"), + ]; + for (const hookPath of hookPaths) { + if (!existsSync(hookPath)) continue; + if (hookPath.endsWith("settings.json")) { + try { + const content = readFileSync(hookPath, "utf-8"); + if (/hooks/i.test(content)) return true; + } catch { + // ignore + } + } else { + return true; + } + } + return false; +} + +function detectBackPressure(targetDir: string, pkgJsonPath: string): boolean { + // JS/TS: package.json scripts + if (existsSync(pkgJsonPath)) { + try { + const pkg = JSON.parse(readFileSync(pkgJsonPath, "utf-8")); + const scripts = pkg.scripts || {}; + const hasTest = "test" in scripts; + const hasTypecheck = "typecheck" in scripts || "type-check" in scripts || "tsc" in scripts; + if (hasTest && hasTypecheck) return true; + } catch { + // ignore + } + } + + // Makefile: test target + const makefilePath = join(targetDir, "Makefile"); + if (existsSync(makefilePath)) { + try { + const content = readFileSync(makefilePath, "utf-8"); + if (/^test\s*:/m.test(content) && (/^lint\s*:/m.test(content) || /^check\s*:/m.test(content))) return true; + } catch { + // ignore + } + } + + // Python: pyproject.toml with test framework + const pyprojectPath = join(targetDir, "pyproject.toml"); + if (existsSync(pyprojectPath)) { + try { + const content = readFileSync(pyprojectPath, "utf-8"); + const hasTestTool = /\[tool\.(pytest|mypy|pyright)\]/i.test(content); + if (hasTestTool) return true; + } catch { + // ignore + } + } + + // Rust: Cargo.toml (Rust has built-in test + type checking) + if (existsSync(join(targetDir, "Cargo.toml"))) return true; + + return false; +} + +function countMcpTools(targetDir: string): number { + const mcpPaths = [ + join(targetDir, ".claude", "mcp.json"), + join(targetDir, ".cursor", "mcp.json"), + join(targetDir, "mcp.json"), + ]; + for (const mcpPath of mcpPaths) { + if (!existsSync(mcpPath)) continue; + try { + const config = JSON.parse(readFileSync(mcpPath, "utf-8")); + const servers = config.mcpServers || config.servers || config; + if (typeof servers === "object" && !Array.isArray(servers)) { + return Object.keys(servers).length; + } + } catch { + // ignore + } + } + return 0; +} + +function loadCustomChecks(targetDir: string): CustomCheck[] { + const checksPath = join(targetDir, ".reins", "custom-checks.json"); + if (!existsSync(checksPath)) return []; + try { + const config = JSON.parse(readFileSync(checksPath, "utf-8")); + if (!Array.isArray(config.checks)) return []; + return config.checks.filter( + (c: unknown): c is CustomCheck => + typeof c === "object" && + c !== null && + "name" in c && + "type" in c && + "path" in c && + "dimension" in c && + ((c as CustomCheck).type === "file-exists" || (c as CustomCheck).type === "file-contains"), + ); + } catch { + return []; + } +} + function detectFrameworks(targetDir: string): string[] { const detected: string[] = []; if (existsSync(join(targetDir, ".codex")) || existsSync(join(targetDir, "WORKFLOW.md"))) detected.push("symphony"); @@ -191,6 +312,10 @@ export function buildAuditRuntimeContext(targetDir: string): AuditRuntimeContext checkWorkflowsForMergeProtection(workflowDir); const hasSpecDocument = existsSync(join(targetDir, "SPEC.md")) || existsSync(join(targetDir, "spec.md")); + const hasHooksConfig = detectHooksConfig(targetDir); + const hasBackPressure = detectBackPressure(targetDir, pkgJsonPath); + const mcpToolCount = countMcpTools(targetDir); + const customChecks = loadCustomChecks(targetDir); return { targetDir, @@ -222,6 +347,10 @@ export function buildAuditRuntimeContext(targetDir: string): AuditRuntimeContext hasConcurrencyLimits, hasMergeProtection, hasSpecDocument, + hasHooksConfig, + hasBackPressure, + mcpToolCount, + customChecks, frameworksDetected: detectFrameworks(targetDir), }; } diff --git a/cli/reins/src/lib/audit/scoring.ts b/cli/reins/src/lib/audit/scoring.ts index 2e45234..fe851b7 100644 --- a/cli/reins/src/lib/audit/scoring.ts +++ b/cli/reins/src/lib/audit/scoring.ts @@ -10,15 +10,21 @@ function scoreRepositoryAgents(result: AuditResult, ctx: AuditRuntimeContext): v const agentsMdPath = join(ctx.targetDir, "AGENTS.md"); if (existsSync(agentsMdPath)) { const lines = readFileSync(agentsMdPath, "utf-8").split("\n").length; - if (lines <= 150) { + if (lines <= 60) { result.scores.repository_knowledge.score++; - result.scores.repository_knowledge.findings.push(`AGENTS.md exists (${lines} lines)`); + result.scores.repository_knowledge.findings.push(`AGENTS.md exists (${lines} lines, ideal: <60)`); + } else if (lines <= 100) { + result.scores.repository_knowledge.score++; + result.scores.repository_knowledge.findings.push(`AGENTS.md exists (${lines} lines, good: <100)`); + } else if (lines <= 150) { + result.scores.repository_knowledge.score++; + result.scores.repository_knowledge.findings.push(`AGENTS.md exists (${lines} lines, acceptable — consider trimming to <60)`); } else { - result.scores.repository_knowledge.findings.push(`AGENTS.md exists but too long (${lines} lines, target: <150)`); + result.scores.repository_knowledge.findings.push(`AGENTS.md exists but too long (${lines} lines, target: <60)`); } } else { result.scores.repository_knowledge.findings.push("AGENTS.md missing"); - result.recommendations.push("Create AGENTS.md as a concise map (~100 lines) — run 'reins init .'"); + result.recommendations.push("Create AGENTS.md as a concise map (<60 lines) — run 'reins init .'"); } const allAgentsMd = findFiles(ctx.targetDir, /^AGENTS\.md$/); @@ -313,6 +319,15 @@ function scoreToolRegistry(result: AuditResult, ctx: AuditRuntimeContext): void } } +function scoreBackPressure(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.hasBackPressure) { + result.scores.agent_legibility.score++; + result.scores.agent_legibility.findings.push("Back-pressure mechanisms detected (test + typecheck scripts)"); + } else { + result.scores.agent_legibility.findings.push("No back-pressure mechanisms (add test + typecheck scripts)"); + } +} + function scoreAgentLegibility(result: AuditResult, ctx: AuditRuntimeContext): void { if (existsSync(ctx.pkgJsonPath)) { try { @@ -334,6 +349,7 @@ function scoreAgentLegibility(result: AuditResult, ctx: AuditRuntimeContext): vo scoreObservability(result, ctx); scoreDependencyFootprint(result, ctx); scoreToolRegistry(result, ctx); + scoreBackPressure(result, ctx); } function scoreGoldenPrinciples(result: AuditResult, ctx: AuditRuntimeContext): void { @@ -484,11 +500,69 @@ function scoreAgentWorkflowOrchestration(result: AuditResult, ctx: AuditRuntimeC } } +function scoreAgentWorkflowHooks(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.hasHooksConfig) { + result.scores.agent_workflow.score++; + result.scores.agent_workflow.findings.push("Lifecycle hooks configured (back-pressure/verification)"); + } +} + +function scoreMcpToolProliferation(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.mcpToolCount === 0) return; + if (ctx.mcpToolCount > 10) { + result.scores.agent_workflow.findings.push( + `MCP tool proliferation warning: ${ctx.mcpToolCount} servers configured (recommend <10 to avoid reasoning degradation)`, + ); + result.recommendations.push("Reduce MCP tool count — too many tool descriptions degrade agent reasoning quality"); + } else { + result.scores.agent_workflow.findings.push(`MCP tools configured: ${ctx.mcpToolCount} servers (healthy)`); + } +} + +function scoreCustomChecks(result: AuditResult, ctx: AuditRuntimeContext): void { + for (const check of ctx.customChecks) { + const dimension = check.dimension as keyof typeof result.scores; + if (!(dimension in result.scores)) continue; + + if (check.type === "file-exists") { + const fullPath = join(ctx.targetDir, check.path); + if (existsSync(fullPath)) { + result.scores[dimension].findings.push(`[custom] ${check.name}: ${check.path} exists`); + } else { + result.scores[dimension].findings.push(`[custom] ${check.name}: ${check.path} missing`); + } + } else if (check.type === "file-contains" && check.pattern) { + const fullPath = join(ctx.targetDir, check.path); + if (existsSync(fullPath)) { + try { + const content = readFileSync(fullPath, "utf-8"); + let regex: RegExp; + try { + regex = new RegExp(check.pattern, "i"); + } catch { + result.scores[dimension].findings.push(`[custom] ${check.name}: invalid regex pattern "${check.pattern}"`); + continue; + } + if (regex.test(content)) { + result.scores[dimension].findings.push(`[custom] ${check.name}: pattern found in ${check.path}`); + } else { + result.scores[dimension].findings.push(`[custom] ${check.name}: pattern not found in ${check.path}`); + } + } catch { + result.scores[dimension].findings.push(`[custom] ${check.name}: error reading ${check.path}`); + } + } + } + } +} + function scoreAgentWorkflow(result: AuditResult, ctx: AuditRuntimeContext): void { scoreAgentWorkflowConfig(result, ctx); scoreAgentWorkflowGovernance(result, ctx); scoreAgentWorkflowCi(result, ctx); scoreAgentWorkflowBlueprints(result, ctx); + scoreAgentWorkflowHooks(result, ctx); + scoreMcpToolProliferation(result, ctx); detectZteSignals(result, ctx); scoreAgentWorkflowOrchestration(result, ctx); } @@ -565,12 +639,15 @@ export function applyAuditScoring(result: AuditResult, ctx: AuditRuntimeContext) scoreGoldenPrinciples(result, ctx); scoreAgentWorkflow(result, ctx); scoreGarbageCollection(result, ctx); + scoreCustomChecks(result, ctx); } +export const MATURITY_THRESHOLDS = { L0: 6, L1: 12, L2: 18, L3: 21 } as const; + export function resolveMaturityLevel(totalScore: number): string { - if (totalScore <= 5) return "L0: Manual"; - if (totalScore <= 11) return "L1: Inloop"; - if (totalScore <= 16) return "L2: Guided Outloop"; - if (totalScore <= 19) return "L3: Full Outloop"; + if (totalScore <= MATURITY_THRESHOLDS.L0) return "L0: Manual"; + if (totalScore <= MATURITY_THRESHOLDS.L1) return "L1: Inloop"; + if (totalScore <= MATURITY_THRESHOLDS.L2) return "L2: Guided Outloop"; + if (totalScore <= MATURITY_THRESHOLDS.L3) return "L3: Full Outloop"; return "L4: Zero Touch"; } diff --git a/cli/reins/src/lib/commands/compare.ts b/cli/reins/src/lib/commands/compare.ts new file mode 100644 index 0000000..1809e14 --- /dev/null +++ b/cli/reins/src/lib/commands/compare.ts @@ -0,0 +1,86 @@ +import { existsSync, readFileSync } from "node:fs"; +import type { AuditResult } from "../types"; + +interface DimensionDelta { + dimension: string; + before: number; + after: number; + delta: number; + max: number; +} + +interface CompareResult { + command: "compare"; + project: string; + before_score: number; + after_score: number; + score_delta: number; + max_score: number; + before_level: string; + after_level: string; + level_changed: boolean; + dimensions: DimensionDelta[]; + new_findings: string[]; + lost_findings: string[]; +} + +export function runCompare( + currentAuditPath: string, + baselinePath: string, + deps: { runAudit: (path: string) => AuditResult }, +): void { + if (!existsSync(baselinePath)) { + console.error(JSON.stringify({ error: `Baseline file not found: ${baselinePath}` })); + process.exit(1); + } + + let baseline: AuditResult; + try { + baseline = JSON.parse(readFileSync(baselinePath, "utf-8")); + } catch { + console.error(JSON.stringify({ error: `Failed to parse baseline JSON: ${baselinePath}` })); + process.exit(1); + } + + let current: AuditResult; + try { + current = deps.runAudit(currentAuditPath); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.error(JSON.stringify({ error: message })); + process.exit(1); + } + + const dimensions: DimensionDelta[] = []; + const allFindings = { before: new Set(), after: new Set() }; + + for (const key of Object.keys(current.scores) as Array) { + const before = baseline.scores?.[key]?.score ?? 0; + const after = current.scores[key].score; + const max = current.scores[key].max; + dimensions.push({ dimension: key, before, after, delta: after - before, max }); + + for (const f of baseline.scores?.[key]?.findings ?? []) allFindings.before.add(f); + for (const f of current.scores[key].findings) allFindings.after.add(f); + } + + const newFindings = [...allFindings.after].filter((f) => !allFindings.before.has(f)); + const lostFindings = [...allFindings.before].filter((f) => !allFindings.after.has(f)); + + const result: CompareResult = { + command: "compare", + project: current.project, + before_score: baseline.total_score ?? 0, + after_score: current.total_score, + score_delta: current.total_score - (baseline.total_score ?? 0), + max_score: current.max_score, + before_level: baseline.maturity_level ?? "unknown", + after_level: current.maturity_level, + level_changed: (baseline.maturity_level ?? "unknown") !== current.maturity_level, + dimensions, + new_findings: newFindings, + lost_findings: lostFindings, + }; + + console.log(JSON.stringify(result, null, 2)); +} diff --git a/cli/reins/src/lib/commands/evolve.test.ts b/cli/reins/src/lib/commands/evolve.test.ts index 0b7f4d9..60e63e8 100644 --- a/cli/reins/src/lib/commands/evolve.test.ts +++ b/cli/reins/src/lib/commands/evolve.test.ts @@ -17,13 +17,13 @@ describe("runEvolve unit behavior", () => { scores: { repository_knowledge: { score: 3, max: 4, findings: [] }, architecture_enforcement: { score: 3, max: 3, findings: [] }, - agent_legibility: { score: 3, max: 4, findings: [] }, + agent_legibility: { score: 4, max: 5, findings: [] }, golden_principles: { score: 3, max: 3, findings: [] }, - agent_workflow: { score: 3, max: 5, findings: [] }, + agent_workflow: { score: 4, max: 6, findings: [] }, garbage_collection: { score: 2, max: 3, findings: [] }, }, - total_score: 17, - max_score: 22, + total_score: 19, + max_score: 24, maturity_level: "L3: Full Outloop", recommendations: [], frameworks_detected: [], diff --git a/cli/reins/src/lib/commands/evolve.ts b/cli/reins/src/lib/commands/evolve.ts index 8c6432c..8d04883 100644 --- a/cli/reins/src/lib/commands/evolve.ts +++ b/cli/reins/src/lib/commands/evolve.ts @@ -6,6 +6,7 @@ import { recommendAutomationPack, scaffoldAutomationPack, } from "../automation-pack"; +import { MATURITY_THRESHOLDS } from "../audit/scoring"; import type { AuditResult, EvolutionPath, EvolutionStep, InitOptions } from "../types"; type LevelKey = "L0" | "L1" | "L2" | "L3" | "L4"; @@ -103,6 +104,13 @@ const EVOLUTION_PATHS: Record = { "Create WORKFLOW.md with YAML frontmatter (concurrency, sandbox, hooks) and markdown body (agent prompt template). Versions agent behavior alongside code.", automated: true, }, + { + step: 7, + action: "Set up back-pressure mechanisms", + description: + "Add test and typecheck scripts so agents can self-verify their work. Back-pressure correlates strongly with agent success rates.", + automated: false, + }, ], success_criteria: "Most new code is written by agents, not humans.", }, @@ -220,10 +228,10 @@ const EVOLUTION_PATHS: Record = { }; function resolveCurrentLevelKey(totalScore: number): LevelKey { - if (totalScore <= 5) return "L0"; - if (totalScore <= 11) return "L1"; - if (totalScore <= 16) return "L2"; - if (totalScore <= 19) return "L3"; + if (totalScore <= MATURITY_THRESHOLDS.L0) return "L0"; + if (totalScore <= MATURITY_THRESHOLDS.L1) return "L1"; + if (totalScore <= MATURITY_THRESHOLDS.L2) return "L2"; + if (totalScore <= MATURITY_THRESHOLDS.L3) return "L3"; return "L4"; } diff --git a/cli/reins/src/lib/types.ts b/cli/reins/src/lib/types.ts index 79614ee..3bf6357 100644 --- a/cli/reins/src/lib/types.ts +++ b/cli/reins/src/lib/types.ts @@ -17,7 +17,7 @@ export interface AuditResult { garbage_collection: AuditScore; }; total_score: number; - max_score: 22; + max_score: 24; maturity_level: string; recommendations: string[]; frameworks_detected: string[]; diff --git a/docs/design-docs/core-beliefs.md b/docs/design-docs/core-beliefs.md index 0785cfe..0f63298 100644 --- a/docs/design-docs/core-beliefs.md +++ b/docs/design-docs/core-beliefs.md @@ -29,3 +29,7 @@ Pay it down continuously in small increments. Background agents handle cleanup. ## 7. Progressive Disclosure Over Information Dumps Give agents a map (short AGENTS.md) and teach them where to look. Don't overwhelm context with everything at once. + +## 8. Configuration Over Model Capability + +Agent failures are almost never about model intelligence — they're about how well humans configured the harness. When an agent produces poor output, examine the configuration surfaces (instructions, tools, back-pressure, context length) before blaming the model. It's a skill issue, not a model issue. diff --git a/docs/design-docs/symphony-integration.md b/docs/design-docs/symphony-integration.md index a3cd5f4..39a12b2 100644 --- a/docs/design-docs/symphony-integration.md +++ b/docs/design-docs/symphony-integration.md @@ -17,7 +17,7 @@ OpenAI released Symphony (March 2026), an Elixir-based autonomous coding orchest - **Evolve**: New steps at L1->L2 (workflow config), L2->L3 (skills composition), L3->L4 (workspace isolation) - **Init**: Scaffolds WORKFLOW.md and SPEC.md templates - **Output**: New `frameworks_detected` informational field in audit output -- **Total max score**: 22 (was 21) +- **Total max score**: 24 (was 21) ## Trade-offs diff --git a/package.json b/package.json index 62b526e..9d54725 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "dev": "cd cli/reins && bun --watch src/index.ts", "test": "cd cli/reins && bun test", "audit:self": "cd cli/reins && bun src/index.ts audit ../...", - "lint": "bunx @biomejs/biome@1.9.4 check ." + "lint": "bunx @biomejs/biome@1.9.4 check .", + "typecheck": "cd cli/reins && tsc --noEmit" }, "license": "MIT" } diff --git a/skill/Reins/HarnessMethodology.md b/skill/Reins/HarnessMethodology.md index 92bed0c..9fa6113 100644 --- a/skill/Reins/HarnessMethodology.md +++ b/skill/Reins/HarnessMethodology.md @@ -1,6 +1,8 @@ # Harness Engineering Methodology Reference -> Source: OpenAI's "Harness Engineering" (Feb 2026, Ryan Lopopolo) +> Sources: +> - OpenAI's "Harness Engineering" (Feb 2026, Ryan Lopopolo) +> - HumanLayer's "Skill Issue: Harness Engineering for Coding Agents" (Mar 2026, Kyle) ## Philosophy @@ -149,6 +151,25 @@ Product requirements versioned in-repo alongside design docs and execution plans ### 10. i18n as Schema Constraint Internationalization treated as a structural schema constraint rather than an afterthought. Enforced at the type level, not bolted on later. +## Configuration Surfaces (HumanLayer) + +The following configuration surfaces shape agent behavior beyond the repository structure: + +### Back-Pressure Mechanisms +Tests, type checks, and coverage reporting allow agents to verify their own work. These correlate strongly with agent success rates. A repo without back-pressure forces agents to commit blind — adding `test`, `typecheck`, and `build` scripts provides structured self-verification. + +### Hooks +User-defined scripts executing at agent lifecycle events (similar to git hooks). Enable notifications, approvals, integrations, and verification without polluting the agent's reasoning context. Examples: pre-commit validation, post-tool-use checks, session-start context loading. + +### Progressive Disclosure via Skills +Agents access specialized knowledge only when the task demands it, rather than loading everything into the system prompt upfront. Skill registries enable this but carry supply-chain risk — treat skill installation like `npm install` from an unknown source. + +### Sub-Agent Context Isolation +Encapsulating discrete tasks in separate agent sessions prevents context rot. Parents see only prompts and final results, not intermediate reasoning noise. Chroma's research confirms performance degrades measurably at longer context lengths. + +### Context Length Discipline +ETH Zurich tested 138 agentfiles: LLM-generated ones hurt performance, human-written ones helped only minimally. The takeaway: keep agent instructions concise and human-authored. HumanLayer recommends AGENTS.md under 60 lines. + ## Anti-Patterns - One giant AGENTS.md (context starvation, instant rot) @@ -157,3 +178,6 @@ Internationalization treated as a structural schema constraint rather than an af - Manual code review as primary quality gate - Opaque dependencies agents can't reason about - Letting tech debt compound without garbage collection +- Tool proliferation in MCP configs (pushes agents into "the dumb zone" — too many tool descriptions consume context budget and degrade reasoning quality) +- LLM-generated agentfiles (ETH Zurich study shows they hurt performance vs. concise human-written ones) +- Preemptive over-engineering of harness config (start simple, add config only when real failures occur) diff --git a/skill/Reins/SKILL.md b/skill/Reins/SKILL.md index 7b4e97e..f3aa79f 100644 --- a/skill/Reins/SKILL.md +++ b/skill/Reins/SKILL.md @@ -19,7 +19,7 @@ Do not re-implement CLI logic in skill instructions. Always run commands and par Use this skill when the user asks to: - Scaffold repository readiness artifacts (`AGENTS.md`, `ARCHITECTURE.md`, `docs/`, `risk-policy.json`) -- Audit or score agent-readiness/maturity (0-22, maturity levels, weakest dimensions) +- Audit or score agent-readiness/maturity (0-24, maturity levels, weakest dimensions) - Diagnose readiness gaps with `doctor` (pass/fail/warn health checks with prescriptive fixes) - Evolve the repository to the next Reins maturity level - Improve docs-drift/policy-as-code enforcement tied to Reins outputs @@ -56,7 +56,7 @@ All Reins commands output deterministic JSON. **Always parse JSON output** — n # Scaffold harness engineering structure reins init [--name ] [--force] [--pack ] -# Score maturity across 6 dimensions (0-22) +# Score maturity across 6 dimensions (0-24) reins audit # Health check with prescriptive fixes @@ -65,6 +65,9 @@ reins doctor # Roadmap to next maturity level reins evolve [--apply] +# Compare audits across repos or over time +reins compare + # Show usage reins help ``` @@ -73,11 +76,11 @@ reins help | Score | Level | Description | |-------|-------|-------------| -| 0-5 | **L0: Manual** | Traditional engineering, no agent infrastructure | -| 6-11 | **L1: Inloop** | Agents help, but humans still write code | -| 12-16 | **L2: Guided Outloop** | Humans steer, agents execute most code | -| 17-19 | **L3: Full Outloop** | Agents handle full lifecycle with human oversight | -| 20-22 | **L4: Zero Touch** | Agents maintain, clean, and evolve the system | +| 0-6 | **L0: Manual** | Traditional engineering, no agent infrastructure | +| 7-12 | **L1: Inloop** | Agents help, but humans still write code | +| 13-18 | **L2: Guided Outloop** | Humans steer, agents execute most code | +| 19-21 | **L3: Full Outloop** | Agents handle full lifecycle with human oversight | +| 22-24 | **L4: Zero Touch** | Agents maintain, clean, and evolve the system | ## Core Reins Principles diff --git a/skill/Reins/Workflows/Audit.md b/skill/Reins/Workflows/Audit.md index daa63b6..5add194 100644 --- a/skill/Reins/Workflows/Audit.md +++ b/skill/Reins/Workflows/Audit.md @@ -20,13 +20,13 @@ For remediation detail, pair with doctor: "scores": { "repository_knowledge": { "score": 4, "max": 4, "findings": ["AGENTS.md exists (56 lines)", "..."] }, "architecture_enforcement": { "score": 3, "max": 3, "findings": ["..."] }, - "agent_legibility": { "score": 4, "max": 4, "findings": ["..."] }, + "agent_legibility": { "score": 5, "max": 5, "findings": ["..."] }, "golden_principles": { "score": 3, "max": 3, "findings": ["..."] }, - "agent_workflow": { "score": 5, "max": 5, "findings": ["..."] }, + "agent_workflow": { "score": 6, "max": 6, "findings": ["..."] }, "garbage_collection": { "score": 3, "max": 3, "findings": ["..."] } }, - "total_score": 22, - "max_score": 22, + "total_score": 24, + "max_score": 24, "maturity_level": "L4: Zero Touch", "frameworks_detected": ["symphony", "claude-code"], "recommendations": ["Project is well-structured. Consider evolving to next maturity level."] @@ -39,7 +39,7 @@ For remediation detail, pair with doctor: ```bash result=$(cd cli/reins && bun src/index.ts audit ) -# Parse: .total_score (integer 0-22) +# Parse: .total_score (integer 0-24) # Parse: .maturity_level (string like "L4: Zero Touch") ``` @@ -59,7 +59,7 @@ result=$(cd cli/reins && bun src/index.ts audit ) ## Audit Dimensions -Score each dimension from 0 up to its max (dimensions have different maxes: RK 0-4, AE 0-3, AL 0-4, GP 0-3, AW 0-5, GC 0-3): +Score each dimension from 0 up to its max (dimensions have different maxes: RK 0-4, AE 0-3, AL 0-5, GP 0-3, AW 0-6, GC 0-3): - **0** = Not present - **1** = Minimal/ad-hoc - **2** = Structured but incomplete @@ -87,7 +87,7 @@ Score each dimension from 0 up to its max (dimensions have different maxes: RK 0 **Bonus findings:** Linter depth details, enforcement signal count. -### 3. Agent Legibility (0-4) +### 3. Agent Legibility (0-5) | Check | Points | |-------|--------| @@ -108,7 +108,7 @@ Score each dimension from 0 up to its max (dimensions have different maxes: RK 0 **Bonus findings:** Principle count, anti-patterns detected, CI gate count. -### 5. Agent Workflow (0-5) +### 5. Agent Workflow (0-6) | Check | Points | |-------|--------| @@ -132,11 +132,11 @@ Score each dimension from 0 up to its max (dimensions have different maxes: RK 0 | Score | Level | Description | |-------|-------|-------------| -| 0-5 | **L0: Manual** | Traditional engineering, no agent infrastructure | -| 6-11 | **L1: Inloop** | Agents help, but humans still write code | -| 12-16 | **L2: Guided Outloop** | Humans steer, agents execute most code | -| 17-19 | **L3: Full Outloop** | Agents handle full lifecycle with human oversight | -| 20-22 | **L4: Zero Touch** | Agents maintain, clean, and evolve the system | +| 0-6 | **L0: Manual** | Traditional engineering, no agent infrastructure | +| 7-12 | **L1: Inloop** | Agents help, but humans still write code | +| 13-18 | **L2: Guided Outloop** | Humans steer, agents execute most code | +| 19-21 | **L3: Full Outloop** | Agents handle full lifecycle with human oversight | +| 22-24 | **L4: Zero Touch** | Agents maintain, clean, and evolve the system | ## Steps From 7a21ca2b6a9074ec89588909c72100251209b963 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 14 Mar 2026 13:50:51 +0000 Subject: [PATCH 2/4] chore: bump cli version to v0.1.5 --- cli/reins/package.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/reins/package.json b/cli/reins/package.json index 751bc7e..e8865d5 100644 --- a/cli/reins/package.json +++ b/cli/reins/package.json @@ -1,6 +1,6 @@ { "name": "reins-cli", - "version": "0.1.4", + "version": "0.1.5", "description": "Scaffold, audit, and evolve projects using the Harness Engineering methodology", "type": "module", "license": "MIT", diff --git a/package.json b/package.json index 9d54725..fa90b4d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "reins", - "version": "0.1.4", + "version": "0.1.5", "description": "Harness Engineering CLI — scaffold, audit, evolve, and doctor agent-readiness", "scripts": { "dev": "cd cli/reins && bun --watch src/index.ts", From 49c9a3f066d5e6be707d34d08c27fb39a7edfa29 Mon Sep 17 00:00:00 2001 From: WellDunDun <45949032+WellDunDun@users.noreply.github.com> Date: Sat, 14 Mar 2026 17:42:00 +0300 Subject: [PATCH 3/4] fix: add ReDoS guard for user-supplied regex in custom checks Reject patterns with nested quantifiers (e.g. (a+)+$) before constructing RegExp to prevent catastrophic backtracking. Co-Authored-By: Claude Opus 4.6 --- cli/reins/src/lib/audit/scoring.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/reins/src/lib/audit/scoring.ts b/cli/reins/src/lib/audit/scoring.ts index fe851b7..24b1078 100644 --- a/cli/reins/src/lib/audit/scoring.ts +++ b/cli/reins/src/lib/audit/scoring.ts @@ -536,6 +536,13 @@ function scoreCustomChecks(result: AuditResult, ctx: AuditRuntimeContext): void if (existsSync(fullPath)) { try { const content = readFileSync(fullPath, "utf-8"); + // Guard against ReDoS: reject nested quantifiers like (a+)+ + if (/([+*?]\)?[+*?]|(\.\*){3,})/.test(check.pattern)) { + result.scores[dimension].findings.push( + `[custom] ${check.name}: regex pattern rejected (nested quantifiers may cause ReDoS)`, + ); + continue; + } let regex: RegExp; try { regex = new RegExp(check.pattern, "i"); From a545379938da68bb8a5cee21277d4ed03e8fb655 Mon Sep 17 00:00:00 2001 From: WellDunDun <45949032+WellDunDun@users.noreply.github.com> Date: Sat, 14 Mar 2026 17:50:36 +0300 Subject: [PATCH 4/4] refactor: extract custom check handlers to reduce cognitive complexity Split scoreCustomChecks into handleFileExistsCheck and handleFileContainsCheck helpers. Resolves Biome lint error (cognitive complexity 25 > 15 limit). Co-Authored-By: Claude Opus 4.6 --- cli/reins/src/lib/audit/scoring.ts | 86 +++++++++++++++++------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/cli/reins/src/lib/audit/scoring.ts b/cli/reins/src/lib/audit/scoring.ts index 24b1078..0055ac9 100644 --- a/cli/reins/src/lib/audit/scoring.ts +++ b/cli/reins/src/lib/audit/scoring.ts @@ -519,46 +519,58 @@ function scoreMcpToolProliferation(result: AuditResult, ctx: AuditRuntimeContext } } +function handleFileExistsCheck( + result: AuditResult, + check: { name: string; path: string; dimension: string }, + ctx: AuditRuntimeContext, +): void { + const dimension = check.dimension as keyof typeof result.scores; + if (!(dimension in result.scores)) return; + const fullPath = join(ctx.targetDir, check.path); + const status = existsSync(fullPath) ? "exists" : "missing"; + result.scores[dimension].findings.push(`[custom] ${check.name}: ${check.path} ${status}`); +} + +function handleFileContainsCheck( + result: AuditResult, + check: { name: string; path: string; dimension: string; pattern?: string }, + ctx: AuditRuntimeContext, +): void { + const dimension = check.dimension as keyof typeof result.scores; + if (!(dimension in result.scores) || !check.pattern) return; + const fullPath = join(ctx.targetDir, check.path); + if (!existsSync(fullPath)) return; + + // Guard against ReDoS: reject nested quantifiers like (a+)+ + if (/([+*?]\)?[+*?]|(\.\*){3,})/.test(check.pattern)) { + result.scores[dimension].findings.push( + `[custom] ${check.name}: regex pattern rejected (nested quantifiers may cause ReDoS)`, + ); + return; + } + + try { + const content = readFileSync(fullPath, "utf-8"); + let regex: RegExp; + try { + regex = new RegExp(check.pattern, "i"); + } catch { + result.scores[dimension].findings.push(`[custom] ${check.name}: invalid regex pattern "${check.pattern}"`); + return; + } + const status = regex.test(content) ? "pattern found" : "pattern not found"; + result.scores[dimension].findings.push(`[custom] ${check.name}: ${status} in ${check.path}`); + } catch { + result.scores[dimension].findings.push(`[custom] ${check.name}: error reading ${check.path}`); + } +} + function scoreCustomChecks(result: AuditResult, ctx: AuditRuntimeContext): void { for (const check of ctx.customChecks) { - const dimension = check.dimension as keyof typeof result.scores; - if (!(dimension in result.scores)) continue; - if (check.type === "file-exists") { - const fullPath = join(ctx.targetDir, check.path); - if (existsSync(fullPath)) { - result.scores[dimension].findings.push(`[custom] ${check.name}: ${check.path} exists`); - } else { - result.scores[dimension].findings.push(`[custom] ${check.name}: ${check.path} missing`); - } - } else if (check.type === "file-contains" && check.pattern) { - const fullPath = join(ctx.targetDir, check.path); - if (existsSync(fullPath)) { - try { - const content = readFileSync(fullPath, "utf-8"); - // Guard against ReDoS: reject nested quantifiers like (a+)+ - if (/([+*?]\)?[+*?]|(\.\*){3,})/.test(check.pattern)) { - result.scores[dimension].findings.push( - `[custom] ${check.name}: regex pattern rejected (nested quantifiers may cause ReDoS)`, - ); - continue; - } - let regex: RegExp; - try { - regex = new RegExp(check.pattern, "i"); - } catch { - result.scores[dimension].findings.push(`[custom] ${check.name}: invalid regex pattern "${check.pattern}"`); - continue; - } - if (regex.test(content)) { - result.scores[dimension].findings.push(`[custom] ${check.name}: pattern found in ${check.path}`); - } else { - result.scores[dimension].findings.push(`[custom] ${check.name}: pattern not found in ${check.path}`); - } - } catch { - result.scores[dimension].findings.push(`[custom] ${check.name}: error reading ${check.path}`); - } - } + handleFileExistsCheck(result, check, ctx); + } else if (check.type === "file-contains") { + handleFileContainsCheck(result, check, ctx); } } }