diff --git a/.gitignore b/.gitignore index 8b53909..38038de 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ .env.local .DS_Store .context/ +MEMORY/ diff --git a/Evaluating AGENTS.md.pdf b/Evaluating AGENTS.md.pdf new file mode 100644 index 0000000..ffd3f3b Binary files /dev/null and b/Evaluating AGENTS.md.pdf differ diff --git a/cli/reins/package.json b/cli/reins/package.json index faa1b96..c468752 100644 --- a/cli/reins/package.json +++ b/cli/reins/package.json @@ -1,6 +1,6 @@ { "name": "reins-cli", - "version": "0.1.2", + "version": "0.1.3", "description": "Scaffold, audit, and evolve projects using the Harness Engineering methodology", "type": "module", "license": "MIT", diff --git a/cli/reins/src/index.test.ts b/cli/reins/src/index.test.ts index 2395d47..bb21e41 100644 --- a/cli/reins/src/index.test.ts +++ b/cli/reins/src/index.test.ts @@ -183,7 +183,7 @@ describe("reins audit", () => { const result = JSON.parse(stdout); expect(result.total_score).toBe(0); - expect(result.max_score).toBe(18); + expect(result.max_score).toBe(21); expect(result.maturity_level).toBe("L0: Manual"); expect(result.scores.repository_knowledge.score).toBe(0); expect(result.scores.architecture_enforcement.score).toBe(0); @@ -268,7 +268,7 @@ describe("reins audit", () => { expect(result).toHaveProperty("max_score"); expect(result).toHaveProperty("maturity_level"); expect(result).toHaveProperty("recommendations"); - expect(result.max_score).toBe(18); + expect(result.max_score).toBe(21); }); test("fails on nonexistent directory", async () => { @@ -331,7 +331,7 @@ describe("reins evolve", () => { const result = JSON.parse(stdout); expect(result.command).toBe("evolve"); expect(result.current_level).toBe("L0: Manual"); - expect(result.next_level).toBe("L1: Assisted"); + expect(result.next_level).toBe("L1: Inloop"); expect(result.steps.length).toBe(5); expect(result.success_criteria).toBeTruthy(); expect(result.weakest_dimensions).toBeDefined(); @@ -347,7 +347,7 @@ describe("reins evolve", () => { // After init, should be at least L1 expect(result.current_level).toMatch(/L1|L2/); if (result.current_level.startsWith("L1")) { - expect(result.next_level).toBe("L2: Steered"); + expect(result.next_level).toBe("L2: Guided Outloop"); } }); diff --git a/cli/reins/src/index.ts b/cli/reins/src/index.ts index cc8e93e..526cc11 100644 --- a/cli/reins/src/index.ts +++ b/cli/reins/src/index.ts @@ -36,11 +36,11 @@ EXAMPLES: reins doctor . # Get prescriptive fixes MATURITY LEVELS: - L0: Manual (0-4) Traditional engineering - L1: Assisted (5-8) Agents help, humans still code - L2: Steered (9-13) Humans steer, agents execute - L3: Autonomous (14-16) Agents handle full lifecycle - L4: Self-Correcting (17-18) System maintains itself + L0: Manual (0-5) Traditional engineering + L1: Inloop (6-10) Agents in the loop, humans code + L2: Guided Outloop (11-15) Humans steer, agents execute + L3: Full Outloop (16-18) Agents handle full lifecycle + L4: Zero Touch (19-21) System maintains itself `; console.log(help); } diff --git a/cli/reins/src/lib/audit/context.ts b/cli/reins/src/lib/audit/context.ts index 8da3fec..92f5d1c 100644 --- a/cli/reins/src/lib/audit/context.ts +++ b/cli/reins/src/lib/audit/context.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { detectCliProject, detectMonorepoWorkspaces, scanWorkflowsForEnforcement } from "../detection"; import { findFiles } from "../filesystem"; @@ -22,22 +22,29 @@ export interface AuditRuntimeContext { isCliRepo: boolean; verifiedDocs: string[]; hasCleanupDocs: boolean; + hasAgentCommands: boolean; + hasSessionOrchestrator: boolean; + hasMcpConfig: boolean; + hasGlobBasedRules: boolean; + hierarchicalAgentContextCount: number; + hasSkillsManifest: boolean; } export function createAuditResult(projectName: string): AuditResult { return { project: projectName, + schema_version: "2.0", timestamp: new Date().toISOString(), scores: { - repository_knowledge: { score: 0, max: 3, findings: [] }, + repository_knowledge: { score: 0, max: 4, findings: [] }, architecture_enforcement: { score: 0, max: 3, findings: [] }, - agent_legibility: { score: 0, max: 3, findings: [] }, + agent_legibility: { score: 0, max: 4, findings: [] }, golden_principles: { score: 0, max: 3, findings: [] }, - agent_workflow: { score: 0, max: 3, findings: [] }, + agent_workflow: { score: 0, max: 4, findings: [] }, garbage_collection: { score: 0, max: 3, findings: [] }, }, total_score: 0, - max_score: 18, + max_score: 21, maturity_level: "L0", recommendations: [], }; @@ -79,6 +86,42 @@ export function buildAuditRuntimeContext(targetDir: string): AuditRuntimeContext const verifiedDocs = readVerifiedDocs(targetDir); const hasCleanupDocs = existsSync(join(targetDir, "docs", "exec-plans", "tech-debt-tracker.md")); + const commandsDir = join(targetDir, ".claude", "commands"); + let hasAgentCommands = false; + if (existsSync(commandsDir)) { + try { + hasAgentCommands = readdirSync(commandsDir).length > 0; + } catch { + // ignore read errors + } + } + + const hasSessionOrchestrator = + existsSync(join(targetDir, ".flow")) || + existsSync(join(targetDir, "gsd")) || + existsSync(join(targetDir, "conductor.json")) || + existsSync(join(targetDir, ".claude", "settings.json")); + + const hasMcpConfig = + existsSync(join(targetDir, ".claude", "mcp.json")) || + existsSync(join(targetDir, ".cursor", "mcp.json")) || + existsSync(join(targetDir, "mcp.json")); + + const hasGlobBasedRules = + existsSync(join(targetDir, ".cursor", "rules")) || + existsSync(join(targetDir, ".claude", "rules")); + + const agentContextFiles = [ + ...findFiles(targetDir, /^AGENTS\.md$/, 3), + ...findFiles(targetDir, /^CLAUDE\.md$/, 3), + ]; + const hierarchicalAgentContextCount = agentContextFiles.length; + + const hasSkillsManifest = + existsSync(join(targetDir, "skills.json")) || + existsSync(join(targetDir, ".claude", "skills")) || + existsSync(join(targetDir, ".cursor", "extensions")); + return { targetDir, pkgJsonPath, @@ -97,5 +140,11 @@ export function buildAuditRuntimeContext(targetDir: string): AuditRuntimeContext isCliRepo, verifiedDocs, hasCleanupDocs, + hasAgentCommands, + hasSessionOrchestrator, + hasMcpConfig, + hasGlobBasedRules, + hierarchicalAgentContextCount, + hasSkillsManifest, }; } diff --git a/cli/reins/src/lib/audit/scoring.ts b/cli/reins/src/lib/audit/scoring.ts index 7de79cc..5ed785b 100644 --- a/cli/reins/src/lib/audit/scoring.ts +++ b/cli/reins/src/lib/audit/scoring.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { detectCliDiagnosabilitySignals } from "../detection"; import { findFiles, safeReadDir } from "../filesystem"; @@ -71,10 +71,25 @@ function scoreRepositoryExecutionPlans(result: AuditResult, ctx: AuditRuntimeCon } } +function scoreConditionalContext(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.hasGlobBasedRules) { + result.scores.repository_knowledge.score++; + result.scores.repository_knowledge.findings.push("Glob-based rule files detected"); + return; + } + if (ctx.hierarchicalAgentContextCount >= 3) { + result.scores.repository_knowledge.score++; + result.scores.repository_knowledge.findings.push( + `Conditional context engineering detected (${ctx.hierarchicalAgentContextCount} hierarchical files)`, + ); + } +} + function scoreRepositoryKnowledge(result: AuditResult, ctx: AuditRuntimeContext): void { scoreRepositoryAgents(result, ctx); scoreRepositoryDocs(result, ctx); scoreRepositoryExecutionPlans(result, ctx); + scoreConditionalContext(result, ctx); } function hasDeepLinterEnforcement(ctx: AuditRuntimeContext): boolean { @@ -288,6 +303,16 @@ function scoreDependencyFootprint(result: AuditResult, ctx: AuditRuntimeContext) } } +function scoreToolRegistry(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.hasMcpConfig || ctx.hasSkillsManifest) { + result.scores.agent_legibility.score++; + const signals: string[] = []; + if (ctx.hasMcpConfig) signals.push("MCP config"); + if (ctx.hasSkillsManifest) signals.push("skills manifest"); + result.scores.agent_legibility.findings.push(`Declared tool registry: ${signals.join("|")}`); + } +} + function scoreAgentLegibility(result: AuditResult, ctx: AuditRuntimeContext): void { if (existsSync(ctx.pkgJsonPath)) { try { @@ -308,6 +333,7 @@ function scoreAgentLegibility(result: AuditResult, ctx: AuditRuntimeContext): vo scoreObservability(result, ctx); scoreDependencyFootprint(result, ctx); + scoreToolRegistry(result, ctx); } function scoreGoldenPrinciples(result: AuditResult, ctx: AuditRuntimeContext): void { @@ -395,10 +421,55 @@ function scoreAgentWorkflowCi(result: AuditResult, ctx: AuditRuntimeContext): vo } } +function scoreAgentWorkflowBlueprints(result: AuditResult, ctx: AuditRuntimeContext): void { + if (ctx.hasAgentCommands || ctx.hasSessionOrchestrator) { + result.scores.agent_workflow.score++; + const signals: string[] = []; + if (ctx.hasAgentCommands) signals.push("commands"); + if (ctx.hasSessionOrchestrator) signals.push("orchestrator"); + result.scores.agent_workflow.findings.push(`Agent blueprint signals: ${signals.join("|")}`); + } +} + +function detectZteSignals(result: AuditResult, ctx: AuditRuntimeContext): void { + if (!existsSync(ctx.workflowDir)) return; + + let hasAutoMerge = false; + let hasDeployOnMerge = false; + + try { + const files = readdirSync(ctx.workflowDir).filter((f) => f.endsWith(".yml") || f.endsWith(".yaml")); + for (const file of files) { + const content = readFileSync(join(ctx.workflowDir, file), "utf-8"); + if (/auto-merge|mergify|kodiak/i.test(content)) { + hasAutoMerge = true; + } + const hasPushToMain = + /push:[\s\S]*?branches:[\s\S]*?(?:-\s*["']?main["']?|\[\s*["']?main["']?\s*\])/i.test(content); + const hasDeploySignal = /\bdeploy(?:ment)?\b/i.test(content); + const hasMergeDeploySignal = /merge[\s\S]*deploy|deploy[\s\S]*merge/i.test(content); + if ((hasPushToMain && hasDeploySignal) || hasMergeDeploySignal) { + hasDeployOnMerge = true; + } + } + } catch { + // ignore read errors + } + + if (hasAutoMerge) { + result.scores.agent_workflow.findings.push("ZTE signal: auto-merge configuration detected"); + } + if (hasDeployOnMerge) { + result.scores.agent_workflow.findings.push("ZTE signal: deploy-on-merge pattern detected"); + } +} + function scoreAgentWorkflow(result: AuditResult, ctx: AuditRuntimeContext): void { scoreAgentWorkflowConfig(result, ctx); scoreAgentWorkflowGovernance(result, ctx); scoreAgentWorkflowCi(result, ctx); + scoreAgentWorkflowBlueprints(result, ctx); + detectZteSignals(result, ctx); } function hasActiveDocGardening(ctx: AuditRuntimeContext): boolean { @@ -476,9 +547,9 @@ export function applyAuditScoring(result: AuditResult, ctx: AuditRuntimeContext) } export function resolveMaturityLevel(totalScore: number): string { - if (totalScore <= 4) return "L0: Manual"; - if (totalScore <= 8) return "L1: Assisted"; - if (totalScore <= 13) return "L2: Steered"; - if (totalScore <= 16) return "L3: Autonomous"; - return "L4: Self-Correcting"; + if (totalScore <= 5) return "L0: Manual"; + if (totalScore <= 10) return "L1: Inloop"; + if (totalScore <= 15) return "L2: Guided Outloop"; + if (totalScore <= 18) return "L3: Full Outloop"; + return "L4: Zero Touch"; } diff --git a/cli/reins/src/lib/commands/evolve.test.ts b/cli/reins/src/lib/commands/evolve.test.ts index 9d256ce..e2e6b34 100644 --- a/cli/reins/src/lib/commands/evolve.test.ts +++ b/cli/reins/src/lib/commands/evolve.test.ts @@ -12,18 +12,19 @@ describe("runEvolve unit behavior", () => { const mockAuditResult: AuditResult = { project: "sample", + schema_version: "2.0", timestamp: new Date().toISOString(), scores: { - repository_knowledge: { score: 3, max: 3, findings: [] }, + repository_knowledge: { score: 3, max: 4, findings: [] }, architecture_enforcement: { score: 3, max: 3, findings: [] }, - agent_legibility: { score: 3, max: 3, findings: [] }, + agent_legibility: { score: 3, max: 4, findings: [] }, golden_principles: { score: 3, max: 3, findings: [] }, - agent_workflow: { score: 1, max: 3, findings: [] }, - garbage_collection: { score: 1, max: 3, findings: [] }, + agent_workflow: { score: 3, max: 4, findings: [] }, + garbage_collection: { score: 2, max: 3, findings: [] }, }, - total_score: 14, - max_score: 18, - maturity_level: "L3: Autonomous", + total_score: 17, + max_score: 21, + maturity_level: "L3: Full Outloop", recommendations: [], }; diff --git a/cli/reins/src/lib/commands/evolve.ts b/cli/reins/src/lib/commands/evolve.ts index afe9821..883566f 100644 --- a/cli/reins/src/lib/commands/evolve.ts +++ b/cli/reins/src/lib/commands/evolve.ts @@ -25,7 +25,7 @@ interface EvolveDeps { const EVOLUTION_PATHS: Record = { L0: { from: "L0: Manual", - to: "L1: Assisted", + to: "L1: Inloop", goal: "Get agents into the development loop", steps: [ { @@ -62,8 +62,8 @@ const EVOLUTION_PATHS: Record = { success_criteria: "Agent can read AGENTS.md, follow pointers, and open a useful PR.", }, L1: { - from: "L1: Assisted", - to: "L2: Steered", + from: "L1: Inloop", + to: "L2: Guided Outloop", goal: "Shift from human-writes-code to human-steers-agent", steps: [ { @@ -100,8 +100,8 @@ const EVOLUTION_PATHS: Record = { success_criteria: "Most new code is written by agents, not humans.", }, L2: { - from: "L2: Steered", - to: "L3: Autonomous", + from: "L2: Guided Outloop", + to: "L3: Full Outloop", goal: "Agent handles full PR lifecycle end-to-end", steps: [ { @@ -119,18 +119,24 @@ const EVOLUTION_PATHS: Record = { }, { step: 3, + action: "Add conditional context engineering", + description: "Create glob-based rule files (.cursor/rules/, .claude/rules/) and per-directory AGENTS.md for targeted agent context.", + automated: false, + }, + { + step: 4, action: "Enable self-validation", description: "Agent drives the app, takes screenshots, checks behavior against expectations.", automated: false, }, { - step: 4, + step: 5, action: "Add doc-gardening automation", description: "Add verification headers (), freshness scripts, and recurring doc review.", automated: false, }, { - step: 5, + step: 6, action: "Build escalation paths", description: "Clear criteria for when to involve humans vs. when agents can proceed autonomously.", automated: false, @@ -139,8 +145,8 @@ const EVOLUTION_PATHS: Record = { success_criteria: "Agent can end-to-end ship a feature from prompt to merge.", }, L3: { - from: "L3: Autonomous", - to: "L4: Self-Correcting", + from: "L3: Full Outloop", + to: "L4: Zero Touch", goal: "System maintains and improves itself without human intervention", steps: [ { @@ -163,12 +169,24 @@ const EVOLUTION_PATHS: Record = { }, { step: 4, + action: "Declare agent tooling registry", + description: "Create MCP config (.claude/mcp.json or mcp.json) and skills manifest declaring available tools for agents.", + automated: false, + }, + { + step: 5, + action: "Implement non-human agent entry points", + description: "Add Slack, API, cron, or webhook triggers so agents can start work without human prompting.", + automated: false, + }, + { + step: 6, action: "Track tech debt continuously", description: "In-repo tracker with recurring review — debt paid down in small increments.", automated: true, }, { - step: 5, + step: 7, action: "Establish docs-drift rules", description: "Link code changes to required doc updates via risk-policy.json watchPaths and docsDriftRules.", automated: false, @@ -179,10 +197,10 @@ const EVOLUTION_PATHS: Record = { }; function resolveCurrentLevelKey(totalScore: number): LevelKey { - if (totalScore <= 4) return "L0"; - if (totalScore <= 8) return "L1"; - if (totalScore <= 13) return "L2"; - if (totalScore <= 16) return "L3"; + if (totalScore <= 5) return "L0"; + if (totalScore <= 10) return "L1"; + if (totalScore <= 15) return "L2"; + if (totalScore <= 18) return "L3"; return "L4"; } @@ -330,7 +348,7 @@ function printL4Response(auditResult: AuditResult): void { project: auditResult.project, current_level: auditResult.maturity_level, current_score: auditResult.total_score, - message: "Already at L4: Self-Correcting. Focus on maintaining quality grades and continuous improvement.", + message: "Already at L4: Zero Touch. Focus on maintaining quality grades and continuous improvement.", pack_recommendation: { recommended: null, reason: "Already at L4. Keep existing automation healthy and continuously verified.", diff --git a/cli/reins/src/lib/types.ts b/cli/reins/src/lib/types.ts index 5fc14b7..fc0187f 100644 --- a/cli/reins/src/lib/types.ts +++ b/cli/reins/src/lib/types.ts @@ -6,6 +6,7 @@ export interface AuditScore { export interface AuditResult { project: string; + schema_version: string; timestamp: string; scores: { repository_knowledge: AuditScore; @@ -16,7 +17,7 @@ export interface AuditResult { garbage_collection: AuditScore; }; total_score: number; - max_score: 18; + max_score: 21; maturity_level: string; recommendations: string[]; } diff --git a/docs/design-docs/ecosystem-positioning.md b/docs/design-docs/ecosystem-positioning.md index 4ba7bb1..232f7f9 100644 --- a/docs/design-docs/ecosystem-positioning.md +++ b/docs/design-docs/ecosystem-positioning.md @@ -4,9 +4,14 @@ How reins relates to other agent-first development tools. ## The Layer Model -Agent-first development operates across three layers. Each layer solves a distinct problem and tools at one layer don't replace tools at another. +Agent-first development operates across four layers. Each layer solves a distinct problem and tools at one layer don't replace tools at another. ``` +Layer 4: AGENT INFRASTRUCTURE + Dedicated agent sandboxes, pre-warmed dev boxes, tool registries. + Tools: Stripe Minions, E2B, Modal, Codex cloud sandboxes. + Problem solved: agent compute isolation, parallel scaling, tool discovery. + Layer 3: SESSION EXECUTION How agents execute work within a single coding session. Tools: GSD, Flow-Next, custom prompt chains. @@ -95,6 +100,10 @@ Repository: https://github.com/gmickel/flow-next-opencode Session orchestrators assume the repo is navigable. Reins makes it navigable. Without clear `AGENTS.md`, documented architecture, and golden principles, session orchestrators are steering agents through a codebase with no map. +### Stripe Minions and Layer 4 + +Stripe's Minions system operates at Layer 3+4 simultaneously — orchestrating agent sessions while managing dedicated compute sandboxes, tool registries, and parallel scaling infrastructure. Their blueprint engines validate the Layer 2 approach that Reins takes: blueprints need documented architecture, golden principles, and agent-legible repos to function. The structural foundation Reins provides at Layer 2 is exactly what systems like Minions depend on to operate effectively at the layers above. + ## The Ideal Stack ``` @@ -102,6 +111,7 @@ reins init . Scaffold structure reins audit . Measure readiness reins evolve . Level up maturity + GSD or Flow-Next Execute coding sessions + + Minions / E2B / Modal Agent infrastructure + CI enforcement of golden principles Continuous quality + Background garbage collection agents Continuous cleanup ``` diff --git a/docs/exec-plans/active/next-iteration-stripe-informed.md b/docs/exec-plans/active/next-iteration-stripe-informed.md new file mode 100644 index 0000000..e41142d --- /dev/null +++ b/docs/exec-plans/active/next-iteration-stripe-informed.md @@ -0,0 +1,259 @@ +# Next Iteration: Stripe-Informed Audit Deepening + +Created: 2026-03-05 +Owner: Team +Status: Active +Source: Stripe Minions agentic engineering analysis (IndyDevDan extract-wisdom, 2026-03-05) + +## Thesis + +Stripe's Minions system validates Reins' core architecture — blueprints, conditional rules, tool registries, and agent sandboxes all map to existing Reins audit dimensions. The next iteration should **deepen existing dimensions** with richer sub-checks rather than adding new dimensions. This keeps the 6-dimension JSON contract stable while increasing the score ceiling from 18 to 21 and dramatically improving signal quality per point. + +## Strategic Approach + +**Don't add dimensions. Raise the ceiling.** + +Current scoring is largely file-existence checks (TD-002). Stripe's patterns reveal what *mature* versions of each dimension look like. The next iteration adds content-aware sub-checks that detect structural quality within existing artifacts, not just their presence. + +**Key constraint preserved:** Zero dependencies, Bun-powered, file-detection only. All new sub-checks use `fs.readFileSync` + regex/string matching on config files already present in repos. + +--- + +## P0 — Ship First (High impact, low risk) + +### WS1: Agent Workflow Blueprints + +**Dimension:** `agent_workflow` (currently max 3) + +**What Stripe validates:** Blueprint engines interleave deterministic steps (lint, test, build) with agent reasoning. The presence of agent-specific workflow definitions — not just generic CI — is a maturity signal. + +**New sub-checks:** + +| Signal | Detection Pattern | Points To | +|--------|------------------|-----------| +| Agent command definitions | `.claude/commands/` dir exists with 1+ files | Agent has custom commands beyond defaults | +| Session orchestrator config | `.flow/`, `gsd/`, `conductor.json`, `.claude/settings.json` | Structured agent workflow beyond ad-hoc prompting | +| Agent-aware CI steps | CI YAML containing `claude`, `codex`, `cursor`, `agent`, `copilot` keywords | CI pipeline includes agent-specific steps | +| Interleaved workflow pattern | CI YAML with lint→test→agent or agent→lint→test ordering | Blueprint pattern: deterministic + agent interleaving | + +**Scoring change:** Increase `agent_workflow.max` from 3 to 4. New sub-check contributes 1 additional point. Total max score becomes 19. + +**Success criteria:** Repos with agent-specific workflow definitions (`.claude/commands/`, `.flow/`, etc.) score higher than repos with only generic CI. + +### WS2: Conditional Context Engineering + +**Dimension:** `repository_knowledge` (currently max 3) + +**What Stripe validates:** Conditional rule files using glob patterns activate context per-directory. This is more sophisticated than a single AGENTS.md — it's hierarchical, conditional, and targeted. + +**New sub-checks:** + +| Signal | Detection Pattern | Points To | +|--------|------------------|-----------| +| Glob-based rule files | `.cursor/rules/*.mdc` with glob frontmatter; `.claude/rules/` or `.claude/settings.json` containing glob patterns | Conditional context engineering | +| Hierarchical agent context | 3+ `AGENTS.md` or `CLAUDE.md` files at different directory depths | Per-package agent context (not monolithic) | +| Context depth score | Count of distinct context files / total directories ratio | Density of agent-discoverable context | + +**Scoring change:** Increase `repository_knowledge.max` from 3 to 4. New sub-check contributes 1 additional point. Total max score becomes 19 (or 20 with WS1). + +**Success criteria:** Repos with hierarchical AGENTS.md and glob-based rule files score higher than repos with a single root AGENTS.md. + +### WS3: Tool Registry Presence + +**Dimension:** `agent_legibility` (currently max 3) + +**What Stripe validates:** A tool shed / meta-tool helps agents discover 500 MCP tools. At smaller scale, the signal is simpler: does the repo declare available tools for agents? + +**New sub-checks:** + +| Signal | Detection Pattern | Points To | +|--------|------------------|-----------| +| MCP configuration | `.claude/mcp.json`, `.cursor/mcp.json`, `mcp.json` | Agent tooling declared and discoverable | +| Tool count depth | MCP config with 3+ tool entries | Meaningful tool ecosystem, not just a stub | +| Skills/extensions manifest | `skills.json`, `.claude/skills/`, `.cursor/extensions/` | Extended agent capabilities registered | + +**Scoring change:** Increase `agent_legibility.max` from 3 to 4. New sub-check contributes 1 additional point. + +**Success criteria:** Repos with MCP configs or skills manifests are recognized as more agent-legible. + +--- + +## P1 — Ship Second (High impact, moderate effort) + +### WS4: Maturity Model Language Update + +**What Stripe validates:** The inloop (human prompting back-and-forth) vs outloop (fully autonomous agent) distinction maps cleanly to L2→L4 progression. Current maturity level descriptions are generic. Stripe's language is sharper. + +**Changes:** + +| Level | Current Description | Updated Description | +|-------|-------------------|-------------------| +| L0: Manual | (score ≤4) | No change — pre-agent baseline | +| L1: Assisted | (score ≤8) | **Inloop** — agents write code within human-driven sessions | +| L2: Steered | (score ≤13) | **Guided Outloop** — agents handle PR lifecycle, humans steer scope | +| L3: Autonomous | (score ≤16) | **Full Outloop** — agents operate from Slack/API triggers without human prompting | +| L4: Self-Correcting | (score >16) | **Zero Touch** — prompt to production, self-healing, system maintains itself | + +**Implementation:** Update `resolveMaturityLevel()` return strings and evolve path descriptions. Non-breaking — the level names (L0-L4) stay the same, only the descriptive suffix changes. + +**Success criteria:** `reins audit` output uses inloop/outloop language. `reins evolve` paths reference the progression clearly. + +### WS5: ZTE Readiness Indicators + +**Dimension:** Spread across existing dimensions as bonus findings (not scored points) + +**What Stripe validates:** Zero Touch Engineering (prompt → production) is the north star. Even before achieving it, repos can show ZTE readiness signals. + +**New findings (informational, not scored):** + +| Signal | Detection Pattern | Finding Text | +|--------|------------------|-------------| +| Auto-merge config | CI YAML with `auto-merge`, `mergify`, `kodiak` keywords | "ZTE signal: auto-merge configuration detected" | +| Bot in CODEOWNERS | `CODEOWNERS` containing bot/agent usernames | "ZTE signal: agent listed as code owner" | +| Deploy-on-merge | CI YAML with deploy step triggered by merge to main | "ZTE signal: deploy-on-merge pattern detected" | +| Agent entry points | Slack/API/webhook triggers for agent workflows | "ZTE signal: non-human entry points for agent work" | + +**These are findings, not score points.** They appear in audit output as forward-looking indicators. When enough accumulate, they naturally inform L4 scoring in a future iteration. + +**Success criteria:** Repos with ZTE-adjacent patterns see them called out in audit findings. + +### WS6: Evolve Path Enrichment + +**Current state:** `reins evolve` paths have 5 steps per level. Stripe's patterns suggest more specific guidance. + +**Changes:** +- L2→L3 path gains a step: "Add conditional context engineering (glob-based rule files, per-directory AGENTS.md)" +- L3→L4 path gains a step: "Declare agent tooling registry (MCP config, skills manifest)" +- L3→L4 path gains a step: "Implement non-human agent entry points (Slack, API, cron triggers)" + +**Success criteria:** `reins evolve` output includes Stripe-validated guidance for conditional context and tool registries. + +--- + +## P2 — Ship Third (Important, larger effort) + +### WS7: Address Tech Debt TD-002 — Beyond File Existence + +**Current state:** Audit heuristics are file-existence only. TD-002 is open. + +**Stripe-informed approach:** Move scoring from "does file X exist?" to "does file X contain meaningful content?" This is what WS1-WS3 already do — they read file contents and check for structural patterns. Generalize this approach across all existing sub-checks: + +| Current Check | Deepened Check | +|---------------|---------------| +| `AGENTS.md exists` | `AGENTS.md exists AND has section headers AND under 150 lines` (already done) | +| `ARCHITECTURE.md exists` | `ARCHITECTURE.md has dependency rules AND domain map table` | +| `Linter config exists` | `Linter config has architectural rules (not just formatting)` (partially done) | +| `CI pipeline exists` | `CI has 3+ enforcement steps AND runs on PRs` | +| `Golden principles exist` | `Golden principles has 5+ rules AND anti-patterns section` | + +**This is an incremental deepening**, not a rewrite. Each existing `existsSync` check gains an optional content-quality bonus. + +**Success criteria:** Audit scores differentiate between "has the file" and "file has meaningful content." + +### WS8: Address Tech Debt TD-003 — Language Detection Broadening + +**Current state:** Scoring is JS/TS-centric (package.json, eslint, biome). TD-003 is open. + +**Stripe-informed insight:** Stripe uses Ruby (hundreds of millions of lines). The audit should detect repo language and adjust scoring signals accordingly. + +**New language detection signals:** + +| Language | Package File | Linter | CI Patterns | +|----------|-------------|--------|-------------| +| Python | `pyproject.toml`, `setup.py`, `requirements.txt` | `ruff.toml`, `.flake8`, `mypy.ini` | `pytest`, `mypy`, `ruff` | +| Ruby | `Gemfile` | `.rubocop.yml` | `rubocop`, `rspec`, `minitest` | +| Go | `go.mod` | `golangci.yml` | `golangci-lint`, `go test` | +| Rust | `Cargo.toml` | `clippy.toml` | `cargo clippy`, `cargo test` | +| Java/Kotlin | `pom.xml`, `build.gradle` | `checkstyle.xml`, `.editorconfig` | `gradle test`, `mvn test` | + +**Implementation:** Add a `detectPrimaryLanguage()` function in `detection.ts`. Scoring functions branch on detected language for linter/CI checks. + +**Success criteria:** `reins audit` produces meaningful scores for Python, Ruby, Go, and Rust repos — not just JS/TS. + +### WS9: Address Tech Debt TD-005 — Plugin System Foundation + +**Current state:** No plugin system for custom audit dimensions. TD-005 is open. + +**Stripe-informed insight:** Stripe's specialization advantage comes from custom tooling. A plugin system lets teams add domain-specific audit checks. + +**Minimal plugin interface:** +```typescript +interface ReinsPlugin { + name: string; + dimension: string; // must be one of the 6 existing dimensions + check(targetDir: string): { score: number; max: number; findings: string[] }; +} +``` + +**Discovery:** `reins audit` looks for `.reins/plugins/*.ts` (or `.js`) files when the `--enable-plugins` flag is passed. Plugins are **never** loaded by default to prevent untrusted code execution when auditing third-party repos. Each plugin exports a `ReinsPlugin`. Plugin scores are added to the relevant dimension's findings (not to the main score — plugins are advisory in v1). + +**Security constraints:** +- Plugins require explicit opt-in via `--enable-plugins` flag +- Discovery restricted to the target repo's `.reins/plugins/` directory only +- Each plugin runs in an isolated child process with a 10-second timeout +- Plugin failures are reported as warnings, never crash the auditor + +**Success criteria:** Teams can write a `.reins/plugins/my-check.ts` file and see its output in `reins audit --enable-plugins`. + +### WS10: Address Tech Debt TD-006 — Compare Command + +**Current state:** Missing compare/watch/self-correct commands. TD-006 is open. + +**Stripe-informed insight:** Stripe iterates with agents by providing feedback after PRs. A `reins compare` command enables before/after audit comparison. + +**Command:** `reins compare ` +- Runs fresh audit against `` +- Diffs scores against `` (previously saved audit output) +- Outputs: which dimensions improved, regressed, or stayed flat + +**Success criteria:** `reins compare old-audit.json .` shows score deltas with directional indicators. + +--- + +## Maturity Score Adjustment + +With P0 changes (WS1-WS3), max score increases from 18 to 21. Maturity thresholds need recalibration: + +| Level | Current Threshold | Proposed Threshold | Rationale | +|-------|------------------|--------------------|-----------| +| L0: Manual | ≤4 | ≤5 | Proportional increase | +| L1: Inloop | ≤8 | ≤10 | Proportional increase | +| L2: Guided Outloop | ≤13 | ≤15 | Proportional increase | +| L3: Full Outloop | ≤16 | ≤18 | Proportional increase | +| L4: Zero Touch | >16 | >18 | Highest bar | + +**Important:** This is a breaking change for consumers comparing scores across versions. The `reins audit` output should include a `schema_version` field so consumers can detect which scoring model produced the result. + +--- + +## Implementation Order + +```text +Phase 1 (P0): WS1 + WS2 + WS3 → deeper sub-checks in 3 dimensions + + score threshold recalibration + schema_version field + Ship as reins-cli v0.2.0 + +Phase 2 (P1): WS4 + WS5 + WS6 → maturity language + ZTE findings + evolve paths + Ship as reins-cli v0.2.1 + +Phase 3 (P2): WS7-WS10 → tech debt resolution + new features + Ship as reins-cli v0.3.0 +``` + +--- + +## Relationship to Active Plans + +- **Skill Eval and Shell Hardening** (active plan): Complementary. That plan focuses on skill quality and evals. This plan focuses on CLI audit depth. No conflicts. WS4-WS6 changes should be covered in skill eval prompt sets. + +--- + +## What We're NOT Doing (Anti-Scope) + +- NOT adding AST analysis (stays file-detection + content regex) +- NOT adding network calls during audit +- NOT adding runtime dependency checking +- NOT building a full tool shed — just detecting if one exists +- NOT implementing agent sandboxes — just detecting sandbox readiness signals +- NOT building Stripe's 500-MCP infrastructure — detecting MCP config presence +- NOT changing the 6-dimension model — deepening within it diff --git a/evaluating-agents-md-review.md b/evaluating-agents-md-review.md new file mode 100644 index 0000000..735c575 --- /dev/null +++ b/evaluating-agents-md-review.md @@ -0,0 +1,199 @@ +# Critique: "Evaluating AGENTS.md: Are Repository-Level Context Files Helpful for Coding Agents?" + +**Paper:** Gloaguen, Mündler, Müller, Raychev, Vechev (ETH Zurich / LogicStar) +**Published:** arXiv:2602.11988v1, 12 Feb 2026 +**Reviewed by:** Daniel Petro (creator of Reins CLI, harness-engineering practitioner) + +--- + +## TL;DR + +The paper asks a good question — do AGENTS.md files actually help coding agents? — but its methodology is too narrow to answer it. It benchmarks context files on isolated bug-fix tasks, finds marginal impact, and concludes context files "are likely only desirable when manually written." That conclusion doesn't follow from the evidence because the evaluation never tests the scenarios where context files matter most: multi-step workflows, architecture decisions, and constraint enforcement. + +--- + +## 1. What the Paper Claims + +1. LLM-generated context files have a **marginal negative** effect on task success (decrease of 0.5% and 2% on SWE-BENCH LITE and AGENTBENCH respectively). +2. Developer-written context files provide only a **marginal performance gain**. +3. Context files consistently **increase the number of steps** and **increase cost** (by 20-25% on average). +4. Context files lead to **more testing and exploration** but don't improve resolution rates. +5. Context files are **redundant** — LLM-generated ones mostly duplicate documentation already available in the repo. +6. Therefore, context files should describe **only minimal requirements** and be **human-written**. + +--- + +## 2. What the Paper Does Well + +### 2.1. Rigorous Experimental Design +The five-stage construction pipeline for AGENTBENCH is well-structured: find repos, filter PRs, set up environments, generate task descriptions, generate unit tests. The use of a standardized task description format with a third-party LLM is a good control. + +### 2.2. Multi-Model Coverage +Testing across Sonnet 4.5, GPT-5.2, GPT-5.1 Mini, and Qwen3-30B with multiple completions per instance gives reasonable breadth across model families. + +### 2.3. Trace Analysis +Section 4.3 is the most valuable part of the paper. The tool-call categorization (build, quality, test, run_exec, search, file_ops, system) and frequency analysis by category is genuinely insightful. The finding that context files increase testing/exploration tokens without improving outcomes is worth investigating further. + +### 2.4. Honest About Limitations +Section 5 acknowledges the Python-only scope, the limited task types, and the need for future work on code quality and security metrics. This is appreciated. + +--- + +## 3. Methodological Problems + +### 3.1. Benchmark Tasks Are the Wrong Evaluation Target + +This is the central flaw. AGENTBENCH consists of **138 isolated bug-fix and feature-addition tasks** derived from pull requests. These are: + +- Self-contained (single PR scope) +- Already described by PR descriptions and issue context +- Solvable by reading the immediate code neighborhood +- Validated by unit tests that check functional correctness only + +Context files like AGENTS.md are not designed for this. They exist to communicate **cross-cutting concerns** that an agent wouldn't discover from a single PR: architectural constraints, module boundaries, testing conventions, deployment pipelines, merge discipline, dependency policies. The paper's own Table 1 shows the average codebase is ~3,337 files — but the tasks only touch tiny slices of those codebases. + +**Analogy:** This is like evaluating a company's employee handbook by measuring whether new hires can fill out a single form faster with or without it. The handbook's value is in preventing systematic errors across hundreds of decisions, not in accelerating individual tasks. + +### 3.2. LLM-Generated Context Files Are a Strawman + +The paper pits three conditions: no context, LLM-generated context, human-written context. But the LLM-generated files are produced by asking models to *summarize the repository* — which produces exactly the kind of redundant overview the paper then criticizes. No serious AGENTS.md practitioner recommends "have GPT summarize your repo." The AGENTS.md spec and community guidance (Anthropic, 2025b; OpenAI, 2025c) explicitly recommend writing **constraints, rules, and workflow instructions** — not summaries. + +Testing auto-generated summaries and concluding "context files don't help" is testing the wrong artifact. + +### 3.3. The "Redundancy" Finding Is Circular + +Section 4.2 finds that LLM-generated context files are redundant with existing repo documentation. Of course they are — they were *generated from* that documentation. This tells us nothing about whether a human-authored AGENTS.md containing information *not already in the repo* (constraints, anti-patterns, architectural decisions) would be redundant. + +### 3.4. Cost Increase Is Expected and Not Inherently Bad + +The paper treats increased steps and cost as a negative signal. But if context files cause agents to run more tests and explore more code (which the trace analysis confirms), that could be *exactly the desired behavior* — trading compute cost for reliability. The paper doesn't measure: + +- Whether the agent's patches introduce fewer regressions +- Whether the patches follow repo conventions +- Whether the patches respect architectural boundaries +- Whether the agent avoids known anti-patterns + +Only measuring pass/fail on unit tests misses the entire quality dimension. + +### 3.5. Sample Size and Repo Selection Bias + +AGENTBENCH has 138 instances from 12 repositories. All are Python. All are open-source. All have existing test suites. This is a specific niche: + +- Excludes polyglot repositories where context files disambiguate tooling +- Excludes enterprise repos with complex CI/CD where constraints matter more +- Excludes repos with minimal documentation (where context files add the most value) +- The paper acknowledges "most repositories containing context files are niche" and "niche repositories have less strict rules regarding pull requests" — this selection bias undermines generalizability + +### 3.6. No Evaluation of Constraint Adherence + +Consider this repo's (Reins) AGENTS.md. It specifies: + +- "Zero external runtime dependencies — stdlib only" +- "All commands output deterministic JSON" +- "Keep CLI command routing in `cli/reins/src/index.ts`; put reusable internals in `cli/reins/src/lib/`" +- "Resolve all PR conversations/comments before merge" + +None of these are testable by unit test pass/fail. An agent could solve a bug perfectly while violating every one of these constraints. The paper's methodology would score that as a success. A real evaluation of AGENTS.md would measure whether agents *follow instructions*, not just whether they produce correct patches. + +--- + +## 4. Statistical Concerns + +### 4.1. Effect Sizes Are Within Noise + +The reported differences (0.5% decrease for LLM, small gains for human) on 138-instance benchmarks with 4-6 completions per model are likely within confidence intervals. The paper doesn't report statistical significance tests, confidence intervals, or p-values. Given the sample sizes, it's plausible that all three conditions produce statistically indistinguishable results — which would mean the paper's conclusions are unsupported. + +### 4.2. Aggregation Hides Per-Repo Variance + +Figure 12 shows per-repository resolution rates. There's substantial variance — some repos show clear improvement with context files, others show degradation. Aggregating across repos with different characteristics (codebase size, documentation quality, task difficulty) obscures whether context files help *for specific types of repos*. + +--- + +## 5. What's Missing from the Evaluation + +| Dimension | Measured? | Why It Matters | +|-----------|-----------|----------------| +| Functional correctness | Yes (unit tests) | Baseline requirement | +| Code quality | No | Style, idioms, conventions | +| Constraint adherence | No | Following explicit rules | +| Architectural alignment | No | Respecting module boundaries | +| Dependency discipline | No | Not adding banned packages | +| Multi-step task performance | No | Real agent workflows | +| Regression prevention | No | Not breaking existing behavior | +| Agent self-correction | No | Following workflows, self-audit | +| Security posture | No | Not introducing vulnerabilities | +| Cost-adjusted quality | No | More steps producing better code | + +The paper measures exactly one of these ten dimensions and draws broad conclusions about the utility of context files. + +--- + +## 6. The Paper's Implicit Model of "Helpful" Is Too Narrow + +The paper frames "helpful" as "increases resolution rate on benchmarks." But practitioners use AGENTS.md for: + +1. **Steering behavior** — "always run tests before committing," "never modify the scoring logic in the skill layer" +2. **Preventing drift** — keeping agents aligned with architectural decisions across sessions +3. **Onboarding acceleration** — giving agents the same context a senior dev would provide in a code review +4. **Constraint enforcement** — making implicit rules explicit and machine-readable + +None of these show up in a unit-test-only benchmark. The paper essentially evaluates whether a map helps you solve a math problem — the answer is "not really," but that says nothing about whether the map helps you navigate. + +--- + +## 7. Positive Takeaways Worth Internalizing + +Despite the above, several findings are useful for practitioners: + +1. **Auto-generated repo summaries don't help.** Stop using them. Write constraints and rules instead. This validates what the AGENTS.md community already recommends. + +2. **Context files increase exploration.** This is actually good for real work — you *want* agents reading more code and running more tests. The paper frames it negatively because their benchmark only rewards speed, not thoroughness. + +3. **Prompt matching matters little.** Figure 9 shows that using CLAUDE CODE's prompt vs. CODEX's prompt makes negligible difference on SWE-BENCH LITE and AGENTBENCH. This suggests context files are model-agnostic, which is good for portability. + +4. **Human-written files outperform LLM-generated ones.** Even in this narrow benchmark, the trend favors human authorship. With a proper evaluation measuring constraint adherence, the gap would likely widen. + +5. **The trace analysis methodology is reusable.** Categorizing tool calls and comparing frequency distributions across conditions is a solid technique for understanding agent behavior changes. + +--- + +## 8. Recommendations for the Authors + +1. **Build a constraint-adherence benchmark.** Define repo-level rules (dependency restrictions, file placement, naming conventions, testing requirements) and measure whether agents follow them with and without context files. + +2. **Test multi-step workflows.** Evaluate on tasks requiring multiple coordinated changes (refactors, feature additions touching multiple modules) where cross-cutting context matters. + +3. **Separate content types.** Test "summary-style" context files vs. "constraint-style" context files vs. "workflow-style" context files independently. The paper lumps them all together. + +4. **Report statistical significance.** Add confidence intervals, paired tests, and effect size measures. The current results may not be statistically meaningful. + +5. **Expand beyond Python.** Test on polyglot repos, Rust/TypeScript/Go projects with stricter typing and build systems where convention adherence is more critical. + +6. **Measure code review outcomes.** Instead of unit tests, have human reviewers evaluate patches on dimensions like convention adherence, readability, and architectural alignment — with and without context files. + +--- + +## 9. Implications for the Reins Project + +This paper inadvertently validates the Reins approach. Reins doesn't just dump a context file — it: + +- **Audits** repos for harness-engineering readiness (scoring rubric) +- **Scaffolds** AGENTS.md with constraints derived from actual repo structure +- **Evolves** context files as repos change +- **Doctors** broken configurations + +The paper shows that naive auto-generation (summarize the repo) doesn't work. Reins' model of structured, constraint-focused, auditable context files is exactly the alternative the paper's own results point toward but don't test. + +--- + +## 10. Final Verdict + +**Score: 5/10** + +Good experimental infrastructure, wrong evaluation target. The paper answers "do context files help agents pass unit tests on isolated bug fixes?" (answer: barely) but markets itself as answering "are repository-level context files helpful for coding agents?" (much broader claim). The gap between the question asked and the question answered is too large for the conclusions to be actionable. + +The paper should be titled: *"LLM-Generated Repository Summaries Do Not Improve Bug-Fix Resolution Rates on Python Benchmarks."* That's a defensible, narrower claim that the evidence supports. + +--- + +*Reviewed 2026-02-25* diff --git a/package.json b/package.json index fae7e92..dbb2310 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "reins", - "version": "0.1.2", + "version": "0.1.3", "description": "Harness Engineering CLI — scaffold, audit, evolve, and doctor agent-readiness", "scripts": { "dev": "cd cli/reins && bun --watch src/index.ts",