Merge pull request #12228 from boxp/fix-task-board-runner-fable-stall

fix(task-board): retry idle Fable runs

Author: boxp

docker/codex-workspace/task-board/task_board_runner.bb

@@ -953,7 +953,7 @@
        "- Delegate long investigation, implementation, file editing, and test execution to Codex whenever practical. If no explicit Codex model is supplied, use the default Codex route: gpt-5.6-terra (GPT-5.5-equivalent, cost-efficient), unless CODEX_TASK_BOARD_MODEL overrides it. Reserve gpt-5.6-sol (via codex-sol/codex-full assignees) for high-complexity tasks. Use the prepared workspace and repository worktrees from this prompt.\n"
        "- If Codex is delegated work, preserve the Task Board runner contract: include a concise delegated-work summary in your final response and end with exactly one TASK_BOARD_RESULT marker that the runner can parse.\n"
        "- For repository changes, make sure a GitHub PR URL is included before returning TASK_BOARD_RESULT: review. If no repository changes were made, include TASK_BOARD_REVIEW_PR: none.\n"
-       "- Progress logging: at each milestone (investigation complete, approach decided, PR created, blocker encountered), append a note to the ticket Notes by running: bb ~/.claude/skills/obsidian-task-board/bin/task-board.bb append-note TICKET_ID --vault \"$CODEX_TASK_BOARD_VAULT\" --source fable --note \"<milestone summary>\"\n\n"))
+       "- Progress logging: at each milestone (investigation complete, approach decided, PR created, blocker encountered), append a note to the ticket Notes by running: bb ~/.claude/skills/obsidian-task-board/bin/task-board.bb append-note TICKET_ID --vault \"$CODEX_TASK_BOARD_VAULT\" --source fable --note \"<milestone summary>\". For lengthy work, log a concise checkpoint before CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS elapses; an entirely idle run is stopped and retried.\n\n"))
 
 (defn codex-sol-policy-prompt [agent]
   (str "High-cost model routing policy:\n"
@@ -971,6 +971,7 @@
                  "~/.claude/skills/obsidian-task-board/bin/task-board.bb"
                  "~/.codex/skills/obsidian-task-board/bin/task-board.bb")]
     (str "Progress logging: at each milestone during your work (investigation complete, approach decided, PR created, blocker encountered), "
+         "and before CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS elapses during lengthy work, "
          "append a note to this ticket's Notes by running:\n"
          "  bb " helper " append-note " ticket-id " --vault \"$CODEX_TASK_BOARD_VAULT\" --source " agent " --note \"<milestone summary>\"\n")))
 
@@ -1055,6 +1056,67 @@
 (defn pr-gate-poll-seconds []
   (env-long "CODEX_TASK_BOARD_PR_GATE_POLL_SECONDS" "15"))
 
+(defn agent-idle-timeout-seconds []
+  ;; Task duration is not a useful failure signal: valid implementation work can
+  ;; take many hours.  Only interrupt an agent when neither its output nor the
+  ;; ticket itself has changed for this long. Set to 0 to disable the watchdog.
+  (let [seconds (env-long "CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS" "7200")]
+    (when (neg? seconds)
+      (throw (ex-info "CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS must not be negative"
+                      {:value seconds})))
+    seconds))
+
+(defn latest-modification-millis [paths]
+  (reduce max 0 (map (fn [path]
+                       (let [file (io/file (str path))]
+                         (if (.exists file) (.lastModified file) 0)))
+                     paths)))
+
+(defn signal-agent-process-group! [process signal]
+  ;; `run-agent!` starts the CLI through `setsid`, making its PID the process
+  ;; group ID. Signalling the negative PID covers all current and subsequently
+  ;; spawned children, including those created while the agent handles TERM.
+  (let [result @(p/process ["/bin/kill" (str "-" signal) "--"
+                            (str "-" (.pid process))]
+                           {:out :string :err :string})]
+    (when-not (zero? (:exit result))
+      (log! (str "failed to send " signal " to idle agent process group: "
+                 (str/trim (:err result)))))))
+
+(defn await-agent! [proc progress-paths idle-timeout-seconds]
+  (let [process (:proc proc)
+        started-at (System/currentTimeMillis)]
+    (loop [last-progress-at (max started-at (latest-modification-millis progress-paths))]
+      (if-not (.isAlive process)
+        {:proc @proc :idle-timeout? false}
+        (do
+          (Thread/sleep 1000)
+          (let [observed-at (latest-modification-millis progress-paths)
+                last-progress-at (max last-progress-at observed-at)
+                idle-millis (- (System/currentTimeMillis) last-progress-at)]
+            (cond
+              ;; The process can finish during the polling sleep. Check again
+              ;; immediately before the timeout action so a valid final response
+              ;; is not discarded as an idle retry.
+              (not (.isAlive process))
+              {:proc @proc :idle-timeout? false}
+
+              (and (pos? idle-timeout-seconds)
+                   (>= idle-millis (* 1000 idle-timeout-seconds)))
+              (do
+                (log! (str "stopping idle agent after " idle-timeout-seconds
+                           " seconds without progress"))
+                (do
+                  (signal-agent-process-group! process "TERM")
+                (Thread/sleep 5000)
+                  ;; Send KILL even if the agent itself exited: its process group
+                  ;; can still contain a TERM-resistant or late-created child.
+                  (signal-agent-process-group! process "KILL")
+                  {:proc @proc :idle-timeout? true}))
+
+              :else
+              (recur last-progress-at))))))))
+
 (defn no-ci-repos []
   ;; Explicit opt-in list of repos (owner/name) known to have no PR CI workflows.
   ;; When a PR belongs to a listed repo and mergeStateStatus=CLEAN with no checks,
@@ -1355,9 +1417,9 @@
     (fs/create-dirs dir)
     (spit (str prompt-path) (prompt-for action ticket-id lane workspace agent))
     (mark-run! ticket-id run :running {:action action :agent agent :lane lane :started-at (now-str)})
-    (let [args (case agent
-                 "fable"
-                 (cond-> ["claude" "--print" "--output-format" "text"]
+    (let [agent-args (case agent
+                       "fable"
+                       (cond-> ["claude" "--print" "--output-format" "text"]
                    (= "true" (env "CODEX_TASK_BOARD_BYPASS_APPROVALS" "true"))
                    (conj "--dangerously-skip-permissions")
 
@@ -1368,7 +1430,7 @@
                    true
                    (into (fable-model-args)))
 
-                 (cond-> ["codex" "exec" "--json" "--cd" (:workspace-dir workspace)
+                       (cond-> ["codex" "exec" "--json" "--cd" (:workspace-dir workspace)
                           "--skip-git-repo-check"
                           "--output-last-message" (str last-message-path)]
                    (= "true" (env "CODEX_TASK_BOARD_BYPASS_APPROVALS" "true"))
@@ -1386,11 +1448,13 @@
 
                    true
                    (conj "-")))
-          proc @(p/process args (cond-> {:in (io/file (str prompt-path))
-                                         :out (io/file (str stdout-path))
-                                         :err (io/file (str stderr-path))}
-                                  (= "fable" agent)
-                                  (assoc :dir (:workspace-dir workspace))))
+          idle-timeout-seconds (agent-idle-timeout-seconds)
+          proc (p/process (into ["setsid"] agent-args) (cond-> {:in (io/file (str prompt-path))
+                                              :out (io/file (str stdout-path))
+                                              :err (io/file (str stderr-path))}
+                                       (= "fable" agent)
+                                       (assoc :dir (:workspace-dir workspace))))
+          {:keys [proc idle-timeout?]} (await-agent! proc [stdout-path stderr-path (ticket-path ticket-id)] idle-timeout-seconds)
           exit (:exit proc)
           _ (when (and (= "fable" agent) (fs/exists? stdout-path))
               (io/copy (io/file (str stdout-path))
@@ -1404,9 +1468,15 @@
                     :agent agent
                     :lane lane
                     :exit-code exit
+                    :idle-timeout? idle-timeout?
                     :result marker
                     :finished-at (now-str)}))
-      {:exit exit :result marker :run-id run :dir (str dir) :last-message last-message})))
+      {:exit exit
+       :result marker
+       :run-id run
+       :dir (str dir)
+       :last-message last-message
+       :idle-timeout? idle-timeout?})))
 
 (defn candidate-action [{:keys [lane status]} assignee]
   (when (supported-assignee? assignee)
@@ -1437,10 +1507,14 @@
       :else
       intended)))
 
-(defn final-note [run-id next-status result last-message review-gate]
+(defn final-note [run-id next-status result last-message review-gate idle-timeout?]
   (let [base (str "Codex task-board run " run-id " finished with result " next-status ".")
         pr-urls (github-pr-urls last-message)]
     (cond
+      idle-timeout?
+      (str base " No agent progress was logged before the idle timeout; "
+           "the run was stopped and will be retried automatically.")
+
       (and (= "blocked" next-status)
            (not (#{"done" "blocked"} result))
            (not (:ok? review-gate)))
@@ -1484,7 +1558,7 @@
                 (move-card! ticket-id "in-progress")
                 (update-frontmatter! ticket-id {:status "in-progress"}))
               (append-note! ticket-id (str "Codex task-board run " (:run-id lock) " started from " lane " with action " (name action) " using " assignee "."))
-              (let [{:keys [exit result run-id dir last-message]} (run-agent! ticket-id action effective-lane assignee lock)
+              (let [{:keys [exit result run-id dir last-message idle-timeout?]} (run-agent! ticket-id action effective-lane assignee lock)
                     intended (cond
                                (not (zero? exit)) "blocked"
                                (= :groom action) "ready"
@@ -1497,7 +1571,18 @@
                                       (record-pr-gate-failure! ticket-id run-id assignee gate-result)
                                       gate-result))
                                   {:ok? true})
-                    next-status (final-status action result exit review-gate)]
+                    next-status (if idle-timeout?
+                                  "in-progress"
+                                  (final-status action result exit review-gate))]
+                (when idle-timeout?
+                  (mark-run! ticket-id run-id :retrying
+                             {:action action
+                              :agent assignee
+                              :lane effective-lane
+                              :exit-code exit
+                              :result result
+                              :idle-timeout? true
+                              :finished-at (now-str)}))
                 (when (= "review" intended)
                   (mark-run! ticket-id run-id (cond
                                                 (:ok? review-gate) :succeeded
@@ -1516,7 +1601,7 @@
                 (update-frontmatter! ticket-id (cond-> {:status next-status
                                                          :assignee (if (= "in-progress" next-status) assignee "boxp")}
                                                   (= "done" next-status) (assoc :closed (today))))
-                (append-note! ticket-id (final-note run-id next-status result last-message review-gate))
+                (append-note! ticket-id (final-note run-id next-status result last-message review-gate idle-timeout?))
                 true)
               (catch Exception e
                 (move-card! ticket-id "blocked")

docs/project_docs/task-board-runner-fable-stall/plan.md

@@ -0,0 +1,23 @@
+# Task Board Runner Fable Stall Fix Plan
+
+## Goal
+
+Prevent Fable-assigned Task Board tickets from remaining in progress indefinitely
+when the agent stops making progress, without imposing a maximum duration on valid
+long-running work.
+
+## Approach
+
+1. Monitor the Fable/Codex process while it is running.
+2. Treat changes to the agent output or ticket file as progress.
+3. Stop only an agent that has produced no progress for the configured idle period
+   (`CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS`, default: 7200 seconds).
+4. Keep the ticket in progress after an idle stop so the next poll retries it.
+5. Preserve explicit `TASK_BOARD_RESULT: blocked` handling for genuine blockers.
+
+## Verification
+
+- Add a regression test for an idle Fable process being retried.
+- Add a regression test that periodic ticket progress prevents an idle timeout.
+- Run the complete Task Board runner test suite with the CI Babashka version.
+- Run Codex review and merge only after its findings are resolved and PR CI passes.

tests/codex-workspace/task-board-runner-test.sh

@@ -99,6 +99,21 @@ fi
 if [[ -n "${CLAUDE_FAKE_START_LOG:-}" ]]; then
   printf '%s %s\n' "${ticket}" "$(date +%s)" >>"${CLAUDE_FAKE_START_LOG}"
 fi
+if [[ -n "${CLAUDE_FAKE_PROGRESS_FILE:-}" ]]; then
+  for _ in $(seq 1 "${CLAUDE_FAKE_PROGRESS_COUNT:-1}"); do
+    sleep "${CLAUDE_FAKE_PROGRESS_INTERVAL:-0}"
+    printf '%s\n' 'fake agent progress' >>"${CLAUDE_FAKE_PROGRESS_FILE}"
+  done
+fi
+if [[ -n "${CLAUDE_FAKE_CHILD_PID_FILE:-}" ]]; then
+  if [[ "${CLAUDE_FAKE_CHILD_IGNORES_TERM:-false}" == true ]]; then
+    (trap '' TERM; sleep "${CLAUDE_FAKE_CHILD_SLEEP:-30}") &
+  else
+    sleep "${CLAUDE_FAKE_CHILD_SLEEP:-30}" &
+  fi
+  printf '%s\n' "$!" >"${CLAUDE_FAKE_CHILD_PID_FILE}"
+  wait "$!"
+fi
 sleep "${CLAUDE_FAKE_SLEEP:-0}"
 printf '%s\n' "${CLAUDE_FAKE_MESSAGE:-TASK_BOARD_RESULT: done}"
 EOF
@@ -299,6 +314,106 @@ test_fable_assignee_runs_via_claude() {
   assert_file_contains "${events}" '^TASK_BOARD_RESULT: done$'
 }
 
+test_fable_agent_idle_timeout_retries() {
+  local tmp vault state bin summary
+  tmp="$(mktemp -d)"
+  vault="${tmp}/vault"
+  state="${tmp}/state"
+  bin="${tmp}/bin"
+  mkdir -p "${bin}"
+  make_fake_claude "${bin}"
+  write_board "${vault}" "- [ ] [[Tickets/BOXP-154|BOXP-154: stalled fable]] #ticket status::in-progress"
+  write_ticket "${vault}" BOXP-154 in-progress fable
+
+  PATH="${bin}:$PATH" \
+    CLAUDE_FAKE_SLEEP=2 \
+    CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS=1 \
+    run_tick "${vault}" "${state}" env >/tmp/task-board-fable-timeout.out
+
+  assert_file_contains "${vault}/Boards/Task Board.md" '\[\[Tickets/BOXP-154\|BOXP-154: stalled fable\]\].*status::in-progress'
+  assert_file_contains "${vault}/Tickets/BOXP-154.md" '^status: in-progress$'
+  assert_file_contains "${vault}/Tickets/BOXP-154.md" 'No agent progress was logged before the idle timeout; the run was stopped and will be retried automatically'
+  summary="$(find "${state}/runs/BOXP-154" -name summary.edn -print | sort | tail -n 1)"
+  assert_file_contains "${summary}" ':idle-timeout\? true'
+
+  PATH="${bin}:$PATH" \
+    CLAUDE_FAKE_SLEEP=2 \
+    CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS=1 \
+    run_tick "${vault}" "${state}" env >/tmp/task-board-fable-timeout-retry.out
+
+  assert_file_contains "${vault}/Boards/Task Board.md" '\[\[Tickets/BOXP-154\|BOXP-154: stalled fable\]\].*status::in-progress'
+  assert_file_contains "${vault}/Tickets/BOXP-154.md" '^status: in-progress$'
+}
+
+test_fable_idle_timeout_stops_agent_children() {
+  local tmp vault state bin child_pid_file child_pid
+  tmp="$(mktemp -d)"
+  vault="${tmp}/vault"
+  state="${tmp}/state"
+  bin="${tmp}/bin"
+  child_pid_file="${tmp}/child.pid"
+  mkdir -p "${bin}"
+  make_fake_claude "${bin}"
+  write_board "${vault}" "- [ ] [[Tickets/BOXP-156|BOXP-156: child process]] #ticket status::in-progress"
+  write_ticket "${vault}" BOXP-156 in-progress fable
+
+  PATH="${bin}:$PATH" \
+    CLAUDE_FAKE_CHILD_PID_FILE="${child_pid_file}" \
+    CLAUDE_FAKE_CHILD_SLEEP=30 \
+    CLAUDE_FAKE_CHILD_IGNORES_TERM=true \
+    CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS=1 \
+    run_tick "${vault}" "${state}" env >/tmp/task-board-fable-child-timeout.out
+
+  child_pid="$(cat "${child_pid_file}")"
+  if kill -0 "${child_pid}" 2>/dev/null; then
+    fail "expected idle timeout to stop Fable child process ${child_pid}"
+  fi
+}
+
+test_invalid_idle_timeout_does_not_start_agent() {
+  local tmp vault state bin start_log
+  tmp="$(mktemp -d)"
+  vault="${tmp}/vault"
+  state="${tmp}/state"
+  bin="${tmp}/bin"
+  start_log="${tmp}/starts.log"
+  mkdir -p "${bin}"
+  make_fake_claude "${bin}"
+  write_board "${vault}" "- [ ] [[Tickets/BOXP-158|BOXP-158: invalid timeout]] #ticket status::in-progress"
+  write_ticket "${vault}" BOXP-158 in-progress fable
+
+  PATH="${bin}:$PATH" \
+    CLAUDE_FAKE_START_LOG="${start_log}" \
+    CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS=-1 \
+    run_tick "${vault}" "${state}" env >/tmp/task-board-invalid-idle-timeout.out
+
+  [[ ! -e "${start_log}" ]] || fail "expected invalid idle timeout to prevent Fable startup"
+  assert_file_contains "${vault}/Tickets/BOXP-158.md" '^status: blocked$'
+}
+
+test_fable_progress_prevents_idle_timeout() {
+  local tmp vault state bin
+  tmp="$(mktemp -d)"
+  vault="${tmp}/vault"
+  state="${tmp}/state"
+  bin="${tmp}/bin"
+  mkdir -p "${bin}"
+  make_fake_claude "${bin}"
+  write_board "${vault}" "- [ ] [[Tickets/BOXP-155|BOXP-155: long fable]] #ticket status::in-progress"
+  write_ticket "${vault}" BOXP-155 in-progress fable
+
+  PATH="${bin}:$PATH" \
+    CLAUDE_FAKE_PROGRESS_FILE="${vault}/Tickets/BOXP-155.md" \
+    CLAUDE_FAKE_PROGRESS_COUNT=4 \
+    CLAUDE_FAKE_PROGRESS_INTERVAL=0.4 \
+    CLAUDE_FAKE_SLEEP=1 \
+    CODEX_TASK_BOARD_AGENT_IDLE_TIMEOUT_SECONDS=1 \
+    run_tick "${vault}" "${state}" env >/tmp/task-board-fable-progress.out
+
+  assert_file_contains "${vault}/Boards/Task Board.md" '\[\[Tickets/BOXP-155\|BOXP-155: long fable\]\].*status::done'
+  assert_file_contains "${vault}/Tickets/BOXP-155.md" '^status: done$'
+}
+
 test_codex_sol_assignee_includes_delegation_policy() {
   local tmp vault state bin prompt_log summary last_message
   tmp="$(mktemp -d)"
@@ -1726,6 +1841,10 @@ BOARD
 
 test_parallel_codex_runs
 test_fable_assignee_runs_via_claude
+test_fable_agent_idle_timeout_retries
+test_fable_idle_timeout_stops_agent_children
+test_invalid_idle_timeout_does_not_start_agent
+test_fable_progress_prevents_idle_timeout
 test_codex_sol_assignee_includes_delegation_policy
 test_codex_full_assignee_includes_delegation_policy
 test_unsupported_assignee_is_ignored