Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions cmd/e2ebench/outcome.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,27 @@ type outcomeSummary struct {
type outcomePoint struct {
ts int64
exploration, verification, objective, regression, churn int
legacyGain, discriminating, debtAge, blindMutations int
discriminating, debtAge, blindMutations int
ebmEligible, ebmFired bool
governorEligible, governorEngaged bool
}

// claimedProgress reports a round that bought something the host could name.
// It replaced the retired novelty scorer's verdict as the predicate the
// false-progress analysis prices against objective transitions; bookkeeping
// rounds, which that scorer credited, correctly no longer claim anything.
func (p outcomePoint) claimedProgress() bool {
return p.exploration > 0 || p.churn > 0 || p.discriminating > 0
}

// verifyPoint is one backfilled verification-transition observation.
type verifyPoint struct {
ts int64
objective, regression int
}

// falseProgressWindow bounds how many later rounds may redeem a legacy
// progress claim with an objective transition before the round counts false.
// falseProgressWindow bounds how many later rounds may redeem a round's
// progress claim with an objective transition before it counts false.
const falseProgressWindow = 3

// observeVerification folds one verification-classified shell result into the
Expand Down Expand Up @@ -118,7 +126,7 @@ func summarizeOutcomePoints(points []outcomePoint, firstTS, lastTS int64) *outco
o.DebtAgeMax = max(o.DebtAgeMax, p.debtAge)
o.Objective += p.objective
o.Regression += p.regression
if p.legacyGain > 0 {
if p.claimedProgress() {
o.ProgressRounds++
}
// The solution stall clock only starts once the run enters its solution
Expand All @@ -142,7 +150,7 @@ func summarizeOutcomePoints(points []outcomePoint, firstTS, lastTS int64) *outco
}
if verifying {
for i, p := range points {
if p.legacyGain <= 0 {
if !p.claimedProgress() {
continue
}
redeemed := false
Expand Down
33 changes: 18 additions & 15 deletions cmd/e2ebench/outcome_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ func writeTrajectory(t *testing.T, name string, lines []string) string {
func TestSummarizeOutcomeFromRecordedShadowSamples(t *testing.T) {
path := writeTrajectory(t, "shadow.trajectory.jsonl", []string{
`{"seq":1,"ts":500,"event":{"kind":"turn_started"}}`,
`{"seq":2,"ts":1000,"outcome_progress":{"round":1,"exploration":1,"legacy_gain":1}}`,
`{"seq":3,"ts":2000,"outcome_progress":{"round":2,"verification":1,"legacy_gain":2}}`,
`{"seq":4,"ts":3000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3}}`,
`{"seq":5,"ts":4000,"outcome_progress":{"round":4,"verification":1,"objective":1,"legacy_gain":2}}`,
`{"seq":6,"ts":5000,"outcome_progress":{"round":5,"churn":1,"legacy_gain":3}}`,
`{"seq":7,"ts":6000,"outcome_progress":{"round":6,"verification":1,"regression":1}}`,
`{"seq":2,"ts":1000,"outcome_progress":{"round":1,"exploration":1}}`,
`{"seq":3,"ts":2000,"outcome_progress":{"round":2,"verification":1,"discriminating":1}}`,
`{"seq":4,"ts":3000,"outcome_progress":{"round":3,"churn":1}}`,
`{"seq":5,"ts":4000,"outcome_progress":{"round":4,"verification":1,"objective":1,"discriminating":1}}`,
`{"seq":6,"ts":5000,"outcome_progress":{"round":5,"churn":1}}`,
`{"seq":7,"ts":6000,"outcome_progress":{"round":6,"verification":1,"regression":1,"discriminating":1}}`,
`{"seq":8,"ts":7000,"event":{"kind":"turn_done"}}`,
})
s, err := summarizeTrajectory(path)
Expand All @@ -35,13 +35,16 @@ func TestSummarizeOutcomeFromRecordedShadowSamples(t *testing.T) {
if o == nil || o.Backfilled {
t.Fatalf("outcome = %+v, want recorded (not backfilled)", o)
}
if o.Rounds != 6 || o.ProgressRounds != 5 {
t.Errorf("rounds=%d progress=%d, want 6/5", o.Rounds, o.ProgressRounds)
// Every round here bought something nameable; whether the claim was worth
// anything is what FalseProgressRounds prices.
if o.Rounds != 6 || o.ProgressRounds != 6 {
t.Errorf("rounds=%d progress=%d, want 6/6", o.Rounds, o.ProgressRounds)
}
// Round 5 claimed legacy progress (a mutation) with no objective transition
// inside the redemption window — the false-progress case.
if o.FalseProgressRounds != 1 {
t.Errorf("false progress = %d, want 1", o.FalseProgressRounds)
// Rounds 5 and 6 claimed progress (a mutation, then a check that broke) with
// no objective transition inside the redemption window — the false-progress
// case. Round 6 is the last round, so nothing can redeem it.
if o.FalseProgressRounds != 2 {
t.Errorf("false progress = %d, want 2", o.FalseProgressRounds)
}
if o.SolutionStallMax != 2 {
t.Errorf("solution stall max = %d, want 2", o.SolutionStallMax)
Expand Down Expand Up @@ -101,9 +104,9 @@ func TestSummarizeOutcomeBackfillsFromVerificationReceipts(t *testing.T) {
func TestSummarizeOutcomeTracksDebtAndTTFDC(t *testing.T) {
path := writeTrajectory(t, "debt.trajectory.jsonl", []string{
`{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`,
`{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":1,"legacy_gain":3,"debt_age":1}}`,
`{"seq":3,"ts":3000,"outcome_progress":{"round":2,"exploration":1,"legacy_gain":1,"debt_age":2}}`,
`{"seq":4,"ts":4000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3,"debt_age":3}}`,
`{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":1,"debt_age":1}}`,
`{"seq":3,"ts":3000,"outcome_progress":{"round":2,"exploration":1,"debt_age":2}}`,
`{"seq":4,"ts":4000,"outcome_progress":{"round":3,"churn":1,"debt_age":3}}`,
`{"seq":5,"ts":9000,"outcome_progress":{"round":4,"discriminating":1,"verification":1}}`,
`{"seq":6,"ts":10000,"event":{"kind":"turn_done"}}`,
})
Expand Down
4 changes: 2 additions & 2 deletions cmd/e2ebench/phasetrace.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ type phaseTrace struct {
Recovery phaseModel `json:"recovery"`

CompactionMs int64 `json:"compaction_ms,omitempty"`
// NoProgressSignals counts progress-guard escalations (each threshold
// fires once), not raw zero-evidence-gain rounds.
// NoProgressSignals counts host runway observations — the low-balance
// updates plus the one spent transition — not rounds without progress.
NoProgressSignals int `json:"no_progress_signals,omitempty"`
// Rounds is the outcome ledger: total classified rounds, how many bought
// progress, the wasted gap total, and the per-outcome composition.
Expand Down
3 changes: 1 addition & 2 deletions cmd/e2ebench/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ type serveRound struct {
Objective int `json:"o,omitempty"`
Regression int `json:"r,omitempty"`
Churn int `json:"c,omitempty"`
Legacy int `json:"g,omitempty"`
}

func runServeMode(dir, suite, addr string) error {
Expand Down Expand Up @@ -106,7 +105,7 @@ func collectServeState(dir string) (*serveState, error) {
TS: p.ts,
Exploration: p.exploration, Verification: p.verification,
Objective: p.objective, Regression: p.regression,
Churn: p.churn, Legacy: p.legacyGain,
Churn: p.churn,
})
}
state.Tasks = append(state.Tasks, t)
Expand Down
3 changes: 1 addition & 2 deletions cmd/e2ebench/trajectory.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,6 @@ type trajectoryRecord struct {
Objective int `json:"objective"`
Regression int `json:"regression"`
Churn int `json:"churn"`
LegacyGain int `json:"legacy_gain"`
Discriminating int `json:"discriminating"`
DebtAge int `json:"debt_age"`
BlindMutations int `json:"blind_mutations"`
Expand Down Expand Up @@ -382,7 +381,7 @@ func (t *trajScan) record(rec trajectoryRecord) {
t.outcomePoints = append(t.outcomePoints, outcomePoint{
ts: rec.TS, exploration: op.Exploration, verification: op.Verification,
objective: op.Objective, regression: op.Regression, churn: op.Churn,
legacyGain: op.LegacyGain, discriminating: op.Discriminating, debtAge: op.DebtAge,
discriminating: op.Discriminating, debtAge: op.DebtAge,
blindMutations: op.BlindMutations, ebmEligible: op.EBMEligible, ebmFired: op.EBMFired,
governorEligible: op.GovernorEligible, governorEngaged: op.GovernorEngaged,
})
Expand Down
4 changes: 2 additions & 2 deletions docs/GOAL_ENFORCEMENT.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Reasonix 的 Goal 模式(`/goal`)将目标推进(Goal)、验收(Delive
| 完成校验 | 默认 | `complete` 声明必须通过 Delivery readiness(todos、验证、review、签收、能力门禁)才会真正完成;不满足时用缺失项开启下一轮 |
| 完成自述与对账 | `update_goal` 的 `completion` | `complete` 可附带自述:`verified` 命令逐条与本会话真实 receipt 对账,没跑过 / 跑失败 / 早于最后一次改动都记为 unbacked claim;`unverified` 与 `risks` 是宿主推断不出的声明,只增不减,永远不阻塞完成 |
| 独立评审 | 无报告时 | 模型未调用 `update_goal` 时,宿主调用一次独立 bounded evaluator 判定;评审不可用/出错/不确定时安全暂停,绝不默认继续 |
| 执行预算 | 默认 | **默认不设任何上限**:没有轮数预算,也没有单次 Run 的轮次上限。需要为无人值守循环设护栏时,配置 `[agent].goal_token_budget`(整个 Goal 累计 token),越线产出一次总结并进入可恢复的 `budget_spend` 暂停。结构化卡死检测保留:相同宿主失败 3 次或成功轮连续 6 次没有新证据时暂停。跨 turn 无进展仍只做观测 |
| 执行预算 | 默认 | **默认不设任何上限**:没有轮数预算,也没有单次 Run 的轮次上限。需要为无人值守循环设护栏时,配置 `[agent].goal_token_budget`(整个 Goal 累计 token),越线产出一次总结并进入可恢复的 `budget_spend` 暂停。结构化卡死检测保留:相同宿主失败 3 次,或每 turn 的调查额度账户耗尽(证据补充额度,空转扣得最快)时暂停。跨 turn 无进展仍只做观测 |
| 暂停/恢复 | `/goal pause` / `/goal resume` | 暂停保留 Goal、todo、Delivery checkpoint 与运行历史;轮次型暂停恢复时追加一档同类别**轮数**(`budget_extensions` 统计轮次追加次数) |
| 立即阻塞 | `blocked` 报告 | 单个 blocked 报告立即结束目标,不再重复三轮确认 |
| 并行调度 | `parallel_tasks` 工具 | 并发派发多个子 agent,各自独立显示结果 |
Expand Down Expand Up @@ -167,7 +167,7 @@ advanceGoalAfterTurn → 读取 update_goal 报告 + readiness + 预算
├─ 无报告 → evaluator 判定一次(失败则安全暂停)
├─ 外层轮次耗尽 → 安全暂停(blocked + budget_turns)
├─ 单 Run 16 轮耗尽 → 总结后安全暂停(blocked + goal_run_budget)
└─ 3 次相同失败 / 6 次零证据成功轮 → 总结后安全暂停(blocked + goal_stuck)
└─ 3 次相同失败 / 调查额度耗尽 → 总结后安全暂停(blocked + goal_stuck)
```

### 并行调度架构
Expand Down
7 changes: 5 additions & 2 deletions docs/GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -1166,8 +1166,11 @@ resumes with its budget granted again; structural stuck pauses start a fresh
Run), or `/goal pause` to pause a running goal manually. `/goal status` shows
the full runtime summary (turns used, tokens used/limit, requests,
observational no-progress streak, extensions).
Within one Run, three repeated identical host failures or six successful
zero-evidence rounds produce a resumable `goal_stuck` pause. At the end of every goal turn
Within one Run, three repeated identical host failures produce a resumable
`goal_stuck` pause, as does a spent investigation runway — the per-turn account
that every round draws on and that evidence pays back, so a turn that keeps
verifying its work is never cut off while one that keeps repeating itself is
stopped in six rounds. At the end of every goal turn
the model reports its disposition through the structured `update_goal` tool
(continue/complete/blocked); when no report arrives, an independent bounded
evaluator judges the turn once, and any evaluator failure pauses the goal
Expand Down
7 changes: 5 additions & 2 deletions docs/GUIDE.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -900,8 +900,11 @@ Goal 按类别运行在**轮次**预算内:简单目标 10 轮,写入型 20
最多 16 个模型轮次,随后获得一次仅总结响应;若仍未完成则以 `goal_run_budget` 可恢复暂停。
进展按 Goal 范围的新颖性计算:新的读取/搜索
结果、mutation、verification、todo/签收变化和 review 会推进目标;完全相同的工具、参数与
结果重复不会推进。单次 Run 内,相同宿主失败连续 3 次,或成功工具轮连续 6 次没有新证据,
会以 `goal_stuck` 可恢复暂停。跨 Goal turn 的无进展数只做观测,不再按 4/6/10 强制暂停。
结果重复不会推进。单次 Run 内,相同宿主失败连续 3 次会以 `goal_stuck` 可恢复暂停;
调查额度耗尽同理——每个 turn 开一个额度账户(相当于 24 个只读轮),每轮等额扣减,
能证伪的观测(跑测试、跑复现)会赚回数轮,落地一个改动正好付清自己那一轮,
只看但确实有新收获的轮次只花空转轮的四分之一。持续验证或持续改动的 turn 永远不会
被打断,反复空转的 turn 六轮就会停下。跨 Goal turn 的无进展数只做观测,不再按 4/6/10 强制暂停。
累计 token 与真实 provider 请求数仍会统计并展示(便于诊断),但**没有
token 硬上限**,也不会在 provider 请求前做 token 准入拦截。Goal 中只陈述 BUG/崩溃/异常
且未要求分析或禁止修改时,默认按写入型轮数类别。暂停会保留 Goal、todo、Delivery
Expand Down
12 changes: 10 additions & 2 deletions docs/SPEC.md
Original file line number Diff line number Diff line change
Expand Up @@ -517,8 +517,16 @@ func (p Policy) Decide(toolName string, readOnly bool, args json.RawMessage) Dec
Outside goal mode, ordinary prompts never change collaboration mode; the user
must choose Goal or use `/goal` explicitly.
Cross-turn no-progress streaks are observational. Within one Run, three
repeated host failures or six successful zero-evidence rounds trigger a
resumable structural-stuck pause. Token and provider-request totals remain
repeated host failures trigger a resumable structural-stuck pause, as does a
spent investigation runway: each turn opens an account worth 24 look-only
rounds, every round costs the same, an observation that could have refuted
the plan earns several rounds back, a landed change covers its own round, and
a round that found something new costs a quarter of one that found nothing.
A turn that keeps proving things or keeps changing things never runs out; one
that keeps repeating itself is gone in six rounds. Banking is capped at
20 rounds. While the balance is low the host states what it measured — dry
rounds, rounds without acting, balance left — and never instructs; when it
reaches zero the host stops requiring further receipts for the turn. Token and provider-request totals remain
observational and are not request-admission limits.
`/goal clear` removes the active goal. Switching into plan/normal mode clears
the active goal in the desktop UI so the collaboration mode remains one of
Expand Down
8 changes: 4 additions & 4 deletions internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -581,12 +581,12 @@ type Agent struct {
stormSig string
stormCount int

// progress escalates adaptively on consecutive zero-evidence-gain rounds
// (see progress_guard.go); reset with the evidence ledger each turn.
// progress holds the turn's investigation runway — the account every round
// draws on and evidence pays back; reset with the evidence ledger each turn.
progress progressGuard

// outcome shadows progress with an outcome-decomposed scorer whose samples
// only feed trajectory recording; it never influences guard behavior.
// outcome is the turn's only round scorer; its sample feeds the runway, the
// EBM trigger, the reasoning governor and trajectory recording.
outcome *evidence.OutcomeTracker

// taskBudget accumulates spend across every Run continuing one task and
Expand Down
15 changes: 10 additions & 5 deletions internal/agent/goal_boundaries_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package agent

import (
"context"
"fmt"
"reflect"
"testing"

"reasonix/internal/agent/testutil"
"reasonix/internal/event"
"reasonix/internal/evidence"
"reasonix/internal/provider"
"reasonix/internal/tool"
)
Expand Down Expand Up @@ -74,13 +76,16 @@ func TestGoalSameFailurePausesAfterStructuralThreshold(t *testing.T) {
}
}

func TestGoalZeroEvidencePausesAfterSixRepeatedSuccesses(t *testing.T) {
// A Goal whose rounds keep succeeding without buying anything spends its
// runway at the fastest rate and pauses resumably once it is gone.
func TestGoalPausesWhenTheInvestigationRunwayIsSpent(t *testing.T) {
reg := tool.NewRegistry()
reg.Add(fakeTool{name: "read_file", readOnly: true})
turns := make([]testutil.Turn, 0, progressStopStreak+2)
for i := range progressStopStreak + 1 {
rounds := evidence.RunwayStart
turns := make([]testutil.Turn, 0, rounds+1)
for i := range rounds {
turns = append(turns, testutil.Turn{ToolCalls: []provider.ToolCall{{
ID: "same-" + string(rune('a'+i)), Name: "read_file", Arguments: `{"path":"same"}`,
ID: fmt.Sprintf("same-%d", i), Name: "read_file", Arguments: `{"path":"same"}`,
}}})
}
turns = append(turns, testutil.Turn{Text: "The repeated read produced no new evidence."})
Expand All @@ -89,7 +94,7 @@ func TestGoalZeroEvidencePausesAfterSixRepeatedSuccesses(t *testing.T) {
ctx := WithDeliveryExecutionScope(context.Background(), DeliveryExecutionScope{ID: "goal-1", TaskText: "research"})
err := a.Run(ctx, "work")
info, ok := InspectRunPause(err)
if !ok || info.Kind != "goal_stuck" || info.Limit != progressStopStreak || info.Key != "goal zero-evidence rounds" {
if !ok || info.Kind != "goal_stuck" || info.Limit != evidence.RunwayStart || info.Key != "goal investigation runway spent" {
t.Fatalf("pause = %+v ok=%v err=%v", info, ok, err)
}
}
Expand Down
Loading
Loading