Skip to content

Commit 9d0b158

Browse files
committed
feat(ralph): add write heartbeat to detect and break read-only loops (#146)
When Claude spends the entire timeout reading without writing code, the loop wastes 15+ minutes per iteration. The circuit breaker only catches this after 3 full loops (45+ min). Add a filesystem-based write heartbeat monitor that kills the driver early if no project files are modified within WRITE_TIMEOUT_MINUTES (default 8). On timeout, the next loop injects a "START WRITING CODE IMMEDIATELY" system prompt. A new CB_READ_ONLY_TIMEOUT_THRESHOLD (default 2) opens the circuit breaker faster than the general no-progress threshold. Closes #146
1 parent 526be14 commit 9d0b158

8 files changed

Lines changed: 592 additions & 4 deletions

File tree

‎ralph/lib/circuit_breaker.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ CB_NO_PROGRESS_THRESHOLD=${CB_NO_PROGRESS_THRESHOLD:-3} # Open circuit af
2222
CB_SAME_ERROR_THRESHOLD=${CB_SAME_ERROR_THRESHOLD:-5} # Open circuit after N loops with same error
2323
CB_OUTPUT_DECLINE_THRESHOLD=${CB_OUTPUT_DECLINE_THRESHOLD:-70} # Open circuit if output declines by >70%
2424
CB_PERMISSION_DENIAL_THRESHOLD=${CB_PERMISSION_DENIAL_THRESHOLD:-2} # Open circuit after N loops with permission denials (Issue #101)
25+
CB_READ_ONLY_TIMEOUT_THRESHOLD=${CB_READ_ONLY_TIMEOUT_THRESHOLD:-2} # Open circuit after N consecutive read-only timeouts (#146)
2526
CB_COOLDOWN_MINUTES=${CB_COOLDOWN_MINUTES:-30} # Minutes before OPEN → HALF_OPEN auto-recovery (Issue #160)
2627
CB_AUTO_RESET=${CB_AUTO_RESET:-false} # Reset to CLOSED on startup instead of waiting for cooldown
2728

@@ -52,6 +53,7 @@ init_circuit_breaker() {
5253
consecutive_no_progress: 0,
5354
consecutive_same_error: 0,
5455
consecutive_permission_denials: 0,
56+
consecutive_read_only_timeouts: 0,
5557
last_progress_loop: 0,
5658
total_opens: 0,
5759
reason: ""
@@ -92,6 +94,7 @@ init_circuit_breaker() {
9294
consecutive_no_progress: 0,
9395
consecutive_same_error: 0,
9496
consecutive_permission_denials: 0,
97+
consecutive_read_only_timeouts: 0,
9598
last_progress_loop: 0,
9699
total_opens: $total_opens,
97100
reason: "Auto-reset on startup"
@@ -155,6 +158,7 @@ record_loop_result() {
155158
local files_changed=$2
156159
local has_errors=$3
157160
local output_length=$4
161+
local was_read_only_timeout=${5:-false}
158162

159163
init_circuit_breaker
160164

@@ -163,14 +167,23 @@ record_loop_result() {
163167
local consecutive_no_progress=$(echo "$state_data" | jq -r '.consecutive_no_progress' | tr -d '[:space:]')
164168
local consecutive_same_error=$(echo "$state_data" | jq -r '.consecutive_same_error' | tr -d '[:space:]')
165169
local consecutive_permission_denials=$(echo "$state_data" | jq -r '.consecutive_permission_denials // 0' | tr -d '[:space:]')
170+
local consecutive_read_only_timeouts=$(echo "$state_data" | jq -r '.consecutive_read_only_timeouts // 0' | tr -d '[:space:]')
166171
local last_progress_loop=$(echo "$state_data" | jq -r '.last_progress_loop' | tr -d '[:space:]')
167172

168173
# Ensure integers
169174
consecutive_no_progress=$((consecutive_no_progress + 0))
170175
consecutive_same_error=$((consecutive_same_error + 0))
171176
consecutive_permission_denials=$((consecutive_permission_denials + 0))
177+
consecutive_read_only_timeouts=$((consecutive_read_only_timeouts + 0))
172178
last_progress_loop=$((last_progress_loop + 0))
173179

180+
# Track read-only timeouts (#146)
181+
if [[ "$was_read_only_timeout" == "true" ]]; then
182+
consecutive_read_only_timeouts=$((consecutive_read_only_timeouts + 1))
183+
else
184+
consecutive_read_only_timeouts=0
185+
fi
186+
174187
# Detect progress from multiple sources:
175188
# 1. Files changed (git diff)
176189
# 2. Completion signal in response analysis (STATUS: COMPLETE or has_completion_signal)
@@ -249,6 +262,9 @@ record_loop_result() {
249262
if [[ $consecutive_permission_denials -ge $CB_PERMISSION_DENIAL_THRESHOLD ]]; then
250263
new_state="$CB_STATE_OPEN"
251264
reason="Permission denied in $consecutive_permission_denials consecutive loops"
265+
elif [[ $consecutive_read_only_timeouts -ge $CB_READ_ONLY_TIMEOUT_THRESHOLD ]]; then
266+
new_state="$CB_STATE_OPEN"
267+
reason="Read-only timeout in $consecutive_read_only_timeouts consecutive loops (agent stuck reading)"
252268
elif [[ $consecutive_no_progress -ge $CB_NO_PROGRESS_THRESHOLD ]]; then
253269
new_state="$CB_STATE_OPEN"
254270
reason="No progress detected in $consecutive_no_progress consecutive loops"
@@ -267,6 +283,9 @@ record_loop_result() {
267283
if [[ $consecutive_permission_denials -ge $CB_PERMISSION_DENIAL_THRESHOLD ]]; then
268284
new_state="$CB_STATE_OPEN"
269285
reason="Permission denied in $consecutive_permission_denials consecutive loops"
286+
elif [[ $consecutive_read_only_timeouts -ge $CB_READ_ONLY_TIMEOUT_THRESHOLD ]]; then
287+
new_state="$CB_STATE_OPEN"
288+
reason="Read-only timeout in $consecutive_read_only_timeouts consecutive loops (agent stuck reading)"
270289
elif [[ "$has_progress" == "true" ]]; then
271290
new_state="$CB_STATE_CLOSED"
272291
reason="Progress detected, circuit recovered"
@@ -305,6 +324,7 @@ record_loop_result() {
305324
--argjson consecutive_no_progress "$consecutive_no_progress" \
306325
--argjson consecutive_same_error "$consecutive_same_error" \
307326
--argjson consecutive_permission_denials "$consecutive_permission_denials" \
327+
--argjson consecutive_read_only_timeouts "$consecutive_read_only_timeouts" \
308328
--argjson last_progress_loop "$last_progress_loop" \
309329
--argjson total_opens "$total_opens" \
310330
--arg reason "$reason" \
@@ -315,6 +335,7 @@ record_loop_result() {
315335
consecutive_no_progress: $consecutive_no_progress,
316336
consecutive_same_error: $consecutive_same_error,
317337
consecutive_permission_denials: $consecutive_permission_denials,
338+
consecutive_read_only_timeouts: $consecutive_read_only_timeouts,
318339
last_progress_loop: $last_progress_loop,
319340
total_opens: $total_opens,
320341
reason: $reason,
@@ -441,6 +462,7 @@ reset_circuit_breaker() {
441462
consecutive_no_progress: 0,
442463
consecutive_same_error: 0,
443464
consecutive_permission_denials: 0,
465+
consecutive_read_only_timeouts: 0,
444466
last_progress_loop: 0,
445467
total_opens: 0,
446468
reason: $reason

‎ralph/lib/enable_core.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,7 @@ PROJECT_TYPE="${project_type}"
688688
MAX_CALLS_PER_HOUR=100
689689
CLAUDE_TIMEOUT_MINUTES=15
690690
CLAUDE_OUTPUT_FORMAT="json"
691+
WRITE_TIMEOUT_MINUTES=8
691692
692693
# Tool permissions
693694
# Comma-separated list of allowed tools
@@ -707,6 +708,7 @@ BEADS_FILTER="status:open"
707708
CB_NO_PROGRESS_THRESHOLD=3
708709
CB_SAME_ERROR_THRESHOLD=5
709710
CB_OUTPUT_DECLINE_THRESHOLD=70
711+
CB_READ_ONLY_TIMEOUT_THRESHOLD=2
710712
RALPHRCEOF
711713
}
712714

‎ralph/lib/write_heartbeat.sh‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/env bash
2+
# Write heartbeat monitor for Ralph (#146)
3+
# Kills the Claude driver early if no project files are modified within a
4+
# configurable timeout window. Prevents the agent from burning the entire
5+
# CLAUDE_TIMEOUT_MINUTES on read-only exploration loops.
6+
7+
# Start a background monitor that watches for project file modifications.
8+
#
9+
# Args:
10+
# $1 - project_dir: root of the project to monitor
11+
# $2 - claude_pid: PID of the portable_timeout wrapper to kill on timeout
12+
# $3 - timeout_minutes: minutes without a write before killing the driver
13+
# $4 - check_interval: seconds between filesystem checks (default 30)
14+
#
15+
# Env:
16+
# _HEARTBEAT_TIMEOUT_SECS_OVERRIDE: if set, overrides timeout_minutes * 60
17+
# (test-support only)
18+
start_write_heartbeat() {
19+
local project_dir=$1 claude_pid=$2 timeout_minutes=$3 check_interval=${4:-30}
20+
local ralph_dir="$project_dir/.ralph"
21+
local marker="$ralph_dir/.heartbeat_marker"
22+
local timeout_secs
23+
24+
if [[ -n "${_HEARTBEAT_TIMEOUT_SECS_OVERRIDE:-}" ]]; then
25+
timeout_secs=$_HEARTBEAT_TIMEOUT_SECS_OVERRIDE
26+
else
27+
timeout_secs=$((timeout_minutes * 60))
28+
fi
29+
30+
# Clean stale state from previous runs
31+
rm -f "$ralph_dir/.write_heartbeat_triggered" \
32+
"$ralph_dir/.write_heartbeat_seen"
33+
34+
# Timestamp marker — find -newer compares against this
35+
touch "$marker"
36+
37+
(
38+
set +e # Prevent inherited set -e from killing monitor on find errors
39+
trap 'exit 0' TERM
40+
41+
local elapsed=0 write_seen=""
42+
43+
while kill -0 "$claude_pid" 2>/dev/null; do
44+
sleep "$check_interval"
45+
elapsed=$((elapsed + check_interval))
46+
47+
# Check for project file modifications (skip once detected)
48+
if [[ -z "$write_seen" ]]; then
49+
local changed
50+
changed=$(find "$project_dir" \
51+
-path "$project_dir/.ralph" -prune -o \
52+
-path "$project_dir/.git" -prune -o \
53+
-path "$project_dir/node_modules" -prune -o \
54+
-path "$project_dir/.next" -prune -o \
55+
-path "$project_dir/dist" -prune -o \
56+
-path "$project_dir/build" -prune -o \
57+
-path "$project_dir/__pycache__" -prune -o \
58+
-path "$project_dir/.turbo" -prune -o \
59+
-newer "$marker" -type f -print -quit 2>/dev/null)
60+
61+
if [[ -n "$changed" ]]; then
62+
touch "$ralph_dir/.write_heartbeat_seen"
63+
write_seen=true
64+
fi
65+
fi
66+
67+
# Timeout: re-check PID is alive (may have exited during sleep),
68+
# then write marker BEFORE kill to eliminate race with wait
69+
if [[ $elapsed -ge $timeout_secs && -z "$write_seen" ]]; then
70+
if kill -0 "$claude_pid" 2>/dev/null; then
71+
echo "true" > "$ralph_dir/.write_heartbeat_triggered"
72+
kill "$claude_pid" 2>/dev/null # GNU timeout propagates SIGTERM to child
73+
fi
74+
exit 0
75+
fi
76+
done
77+
) &
78+
79+
echo $! > "$ralph_dir/.write_heartbeat_pid"
80+
}
81+
82+
# Stop the heartbeat monitor and clean up marker files.
83+
# Safe to call even if no monitor is running.
84+
stop_write_heartbeat() {
85+
local ralph_dir="${RALPH_DIR:-.ralph}"
86+
local pid_file="$ralph_dir/.write_heartbeat_pid"
87+
88+
if [[ -f "$pid_file" ]]; then
89+
local pid
90+
pid=$(cat "$pid_file")
91+
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
92+
kill "$pid" 2>/dev/null || true
93+
wait "$pid" 2>/dev/null || true
94+
fi
95+
rm -f "$pid_file"
96+
fi
97+
98+
rm -f "$ralph_dir/.heartbeat_marker" "$ralph_dir/.write_heartbeat_seen"
99+
}
100+
101+
# Check whether the most recent execution was killed by the heartbeat.
102+
# Returns 0 (true) if the timeout marker exists.
103+
# Does NOT delete the marker — the caller is responsible for cleanup.
104+
was_write_heartbeat_timeout() {
105+
local ralph_dir="${RALPH_DIR:-.ralph}"
106+
[[ -f "$ralph_dir/.write_heartbeat_triggered" ]]
107+
}

0 commit comments

Comments
 (0)