Merge pull request #61 from alanshurafa/claude/swebench-50task-resilience

Make batch runs resumable, fault-tolerant, and shardable

Author: alanshurafa

benchmarks/code/scripts/evaluate-swebench.sh

@@ -43,10 +43,23 @@ case "$MODE" in
   predictions)
     predictions="${1:-}"
     [[ -n "$predictions" ]] || { code_die "predictions mode needs a JSONL file"; exit 2; }
+    shift || true
+    # The evaluator names its report after the run id and nothing else, so two
+    # benchmark runs of the same condition are indistinguishable afterwards and
+    # a page built from them silently mixes subsets. --label stamps the batch
+    # into the run id so a report can be traced back to the run that made it.
+    LABEL="code-bench"
+    while (( $# > 0 )); do
+      case "$1" in
+        --label) LABEL="${2:?--label needs a value}"; shift 2 ;;
+        *) code_die "unknown evaluate option: $1"; exit 2 ;;
+      esac
+    done
+    [[ "$LABEL" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "--label must be filesystem-safe"; exit 2; }
     bash "$CODE_DIR/validate-predictions.sh" "$predictions" "$SUITE"
     predictions_dir=$(cd "$(dirname "$predictions")" && pwd -P)
     predictions="$predictions_dir/$(basename "$predictions")"
-    run_id="code-bench-$(date -u +%Y%m%dT%H%M%SZ)"
+    run_id="$LABEL-$(date -u +%Y%m%dT%H%M%SZ)"
     (cd "$EVAL_ROOT" && "$CLI" eval verified -p "$predictions" --run-id "$run_id" -j "${CODE_BENCH_EVAL_JOBS:-1}")
     ;;
   *)

benchmarks/code/scripts/run-canary.sh

@@ -10,6 +10,7 @@ RUN_ID=""
 CONDITIONS="A,B,C"
 TASK_LIMIT=1
 TASK=""
+SHARD_SPEC=""
 MAX_CLAUDE=""
 DRY_RUN=false
 SUITE=$(code_suite_id)
@@ -20,6 +21,7 @@ while (( $# > 0 )); do
     --conditions) CONDITIONS="${2:?--conditions needs a value}"; shift 2 ;;
     --task-limit) TASK_LIMIT="${2:?--task-limit needs a value}"; shift 2 ;;
     --task) TASK="${2:?--task needs a value}"; shift 2 ;;
+    --shard) SHARD_SPEC="${2:?--shard needs N/M}"; shift 2 ;;
     --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;;
     --dry-run) DRY_RUN=true; shift ;;
     *) code_die "unknown canary option: $1"; exit 2 ;;
@@ -30,6 +32,21 @@ done
 [[ "$TASK_LIMIT" =~ ^[1-9][0-9]*$ ]] || { code_die "--task-limit must be positive"; exit 2; }
 [[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required"; exit 2; }
 
+# --shard N/M splits the subset across M processes that run at the same time.
+# Shards take disjoint instances, so their cell directories never collide, and
+# each writes its own prediction file because concurrent appends to one file
+# interleave. Wall clock falls by roughly M; provider concurrency rises by M,
+# which is the real limit on how far this can be pushed.
+SHARD_INDEX=0
+SHARD_COUNT=1
+if [[ -n "$SHARD_SPEC" ]]; then
+  [[ "$SHARD_SPEC" =~ ^[0-9]+/[0-9]+$ ]] || { code_die "--shard must look like N/M"; exit 2; }
+  SHARD_INDEX="${SHARD_SPEC%%/*}"
+  SHARD_COUNT="${SHARD_SPEC##*/}"
+  (( SHARD_COUNT >= 1 )) || { code_die "--shard M must be at least 1"; exit 2; }
+  (( SHARD_INDEX < SHARD_COUNT )) || { code_die "--shard N must be less than M"; exit 2; }
+fi
+
 suite_json=$(code_suite_json "$SUITE")
 subset=$(code_subset_path "$suite_json")
 
@@ -54,36 +71,91 @@ fi
 pred_dir="$CODE_BENCH_RESULTS_ROOT/predictions/$RUN_ID"
 mkdir -p "$pred_dir"
 
+runs_root="$CODE_BENCH_RESULTS_ROOT/runs/$RUN_ID"
+failed_cells=0
+done_cells=0
+skipped_cells=0
+
 task_index=0
 while IFS= read -r instance; do
   task_index=$((task_index + 1))
   (( task_index <= TASK_LIMIT )) || break
+  (( (task_index - 1) % SHARD_COUNT == SHARD_INDEX )) || continue
   for condition in $(printf '%s' "$CONDITIONS" | tr ',' ' '); do
-    input=$(bash "$CODE_DIR/scripts/prepare-swebench-instance.sh" "$instance" "$RUN_ID" "$condition")
+    cell="$runs_root/$instance/$condition"
+    if (( SHARD_COUNT > 1 )); then
+      pred_file="$pred_dir/$condition.s$SHARD_INDEX.jsonl"
+    else
+      pred_file="$pred_dir/$condition.jsonl"
+    fi
+    # A fifty-task batch is hours long, so it has to be restartable. A cell that
+    # already holds a prediction is finished and is left alone; a cell without
+    # one is an abandoned clone from an interrupted attempt, and re-preparing it
+    # is the only way forward because prepare refuses an existing directory.
+    if [[ -f "$cell/prediction.json" ]]; then
+      # The cell is the record of truth; the prediction file is a projection of
+      # it. Re-link a finished cell whose line is missing, so a resume under a
+      # different shard layout still yields complete prediction files.
+      if ! jq -e --arg id "$instance" 'select(.instance_id == $id)' "$pred_file" \
+           >/dev/null 2>&1; then
+        jq -c . "$cell/prediction.json" >> "$pred_file"
+      fi
+      printf 'SKIP: %s/%s already has a prediction\n' "$instance" "$condition"
+      skipped_cells=$((skipped_cells + 1)); continue
+    fi
+    [[ ! -d "$cell" ]] || rm -rf "$cell"
+    if ! input=$(bash "$CODE_DIR/scripts/prepare-swebench-instance.sh" "$instance" "$RUN_ID" "$condition"); then
+      printf 'CELL FAILED: could not prepare %s/%s\n' "$instance" "$condition" >&2
+      failed_cells=$((failed_cells + 1)); continue
+    fi
     tier=$(jq -r --arg id "$condition" '.conditions[] | select(.id == $id) | .tier' \
       "$CODE_DIR/conditions.json" | tr -d '\r')
     # Single-shot cells have no agent loop, so they take the other driver. The
     # tier decides, not a hardcoded condition list: a new single-shot arm routes
     # itself the moment conditions.json declares its tier.
+    #
+    # A cell that produces no patch is a zero for that arm, not a reason to
+    # abandon the batch: the subset is the denominator either way, and an abort
+    # here used to throw away every remaining cell in the run.
+    cell_rc=0
     if [[ "$tier" == "single-shot" ]]; then
       agent=$(jq -r --arg id "$condition" \
         '.conditions[] | select(.id == $id) | .dispatches | to_entries
          | map(select(.value > 0)) | .[0].key' \
         "$CODE_DIR/conditions.json" | tr -d '\r')
       bash "$CODE_DIR/drivers/run-single-shot.sh" --input "$input" \
-        --predictions "$pred_dir/$condition.jsonl" --agent "$agent"
+        --predictions "$pred_file" --agent "$agent" || cell_rc=$?
     else
       per_condition=$(jq -r --arg id "$condition" '.conditions[] | select(.id == $id) | .dispatches.claude' \
         "$CODE_DIR/conditions.json" | tr -d '\r')
       bash "$CODE_DIR/drivers/run-workflow.sh" --input "$input" \
-        --predictions "$pred_dir/$condition.jsonl" \
-        --max-claude-dispatches "$per_condition"
+        --predictions "$pred_file" \
+        --max-claude-dispatches "$per_condition" || cell_rc=$?
+    fi
+    if (( cell_rc != 0 )); then
+      printf 'CELL FAILED: %s/%s produced no prediction (rc=%s)\n' "$instance" "$condition" "$cell_rc" >&2
+      failed_cells=$((failed_cells + 1))
+    else
+      done_cells=$((done_cells + 1))
     fi
   done
 done < <(if [[ -n "$TASK" ]]; then printf '%s\n' "$TASK"
          else jq -r '.instances[].instance_id' "$subset" | tr -d '\r'; fi)
 
-for predictions in "$pred_dir"/*.jsonl; do
+# Validate only this shard's own files. A sibling shard is still appending to
+# its own, and validating a file mid-write reports a failure that is not real.
+shopt -s nullglob
+if (( SHARD_COUNT > 1 )); then
+  validate_glob=("$pred_dir"/*.s"$SHARD_INDEX".jsonl)
+else
+  validate_glob=("$pred_dir"/*.jsonl)
+fi
+for predictions in "${validate_glob[@]}"; do
   bash "$CODE_DIR/validate-predictions.sh" "$predictions" "$SUITE"
 done
-printf 'COMPLETE: canary predictions -> %s\n' "$pred_dir"
+printf 'COMPLETE: %s cell(s) generated, %s reused, %s failed -> %s\n' \
+  "$done_cells" "$skipped_cells" "$failed_cells" "$pred_dir"
+if (( failed_cells > 0 )); then
+  printf 'INCOMPLETE: %s cell(s) produced no prediction. They score zero against the\n' "$failed_cells" >&2
+  printf 'subset; rerun the same command to retry only those cells.\n' >&2
+fi

benchmarks/code/tests/test-code-bench.sh

@@ -207,6 +207,55 @@ else
   fail "the canary runner routes single-shot cells to their own driver"
 fi
 
+# A fifty-task batch is hours long. One arm that produces no patch is a zero for
+# that arm, and an abort there discards every remaining cell in the run.
+RESUME_ROOT="$TMP/resume"
+mkdir -p "$RESUME_ROOT/runs/resume-test/pallets__flask-5014/A"
+jq -n '{instance_id:"pallets__flask-5014",model_name_or_path:"co-evolution-condition-A",
+        model_patch:"diff --git a/a.py b/a.py\n"}' \
+  > "$RESUME_ROOT/runs/resume-test/pallets__flask-5014/A/prediction.json"
+resume_out=$(CODE_BENCH_SUITE=swebench-verified-poc CODE_BENCH_RESULTS_ROOT="$RESUME_ROOT" \
+  bash "$RUNNER" run-canary --run-id resume-test --task pallets__flask-5014 \
+  --conditions A --max-claude-dispatches 1 2>&1)
+if printf '%s' "$resume_out" | grep -q 'SKIP: pallets__flask-5014/A already has a prediction' \
+   && printf '%s' "$resume_out" | grep -q '0 cell(s) generated, 1 reused, 0 failed'; then
+  pass "a cell with a prediction is reused instead of rerun"
+else
+  fail "a cell with a prediction is reused instead of rerun"
+fi
+
+# Shards must partition the subset: every instance claimed by exactly one shard,
+# or a parallel run silently double-spends on some tasks and skips others.
+SHARD_ROOT="$TMP/shard"
+while IFS= read -r inst; do
+  mkdir -p "$SHARD_ROOT/runs/shard-test/$inst/A"
+  jq -n --arg id "$inst" '{instance_id:$id,model_name_or_path:"x",model_patch:"diff\n"}' \
+    > "$SHARD_ROOT/runs/shard-test/$inst/A/prediction.json"
+done < <(jq -r '.instances[].instance_id' "$CODE_DIR/subsets/swebench-verified-canary.json" | tr -d '\r')
+shard_total=0
+shard_ok=true
+for s in 0 1; do
+  out=$(CODE_BENCH_RESULTS_ROOT="$SHARD_ROOT" bash "$RUNNER" run-canary \
+    --run-id shard-test --shard "$s/2" --task-limit 5 --conditions A \
+    --max-claude-dispatches 5 2>&1)
+  n=$(printf '%s' "$out" | grep -c '^SKIP: ')
+  shard_total=$((shard_total + n))
+  # 5 instances over 2 shards is 3 and 2; neither may be empty or take them all.
+  if (( n == 0 || n == 5 )); then shard_ok=false; fi
+done
+if [[ "$shard_ok" == true ]] && (( shard_total == 5 )); then
+  pass "shards partition the subset exactly once each"
+else
+  fail "shards partition the subset exactly once each (total=$shard_total)"
+fi
+
+if bash "$RUNNER" run-canary --run-id bad-shard --shard 3/2 --conditions A \
+     --max-claude-dispatches 1 --dry-run >/dev/null 2>&1; then
+  fail "a shard index outside its count is rejected"
+else
+  pass "a shard index outside its count is rejected"
+fi
+
 if jq -e 'all(.conditions[]; (.tier == "agentic") or (.tier == "single-shot"))' \
      "$CODE_DIR/conditions.json" >/dev/null 2>&1; then
   pass "every condition declares a tier"

benchmarks/site/aggregate.sh

@@ -10,11 +10,13 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
 RESULTS_ROOT="${CODE_BENCH_RESULTS_ROOT:-$REPO_ROOT/benchmarks/results/code}"
 SUITE="swebench-verified-canary"
+RUN_LABEL=""
 OUTPUT="$RESULTS_ROOT/site/leaderboard.json"
 
 while (( $# > 0 )); do
   case "$1" in
     --suite) SUITE="${2:?--suite needs a value}"; shift 2 ;;
+    --run-label) RUN_LABEL="${2:?--run-label needs a value}"; shift 2 ;;
     --output) OUTPUT="${2:?--output needs a value}"; shift 2 ;;
     *) printf 'ERROR: unknown aggregate option: %s\n' "$1" >&2; exit 2 ;;
   esac
@@ -23,11 +25,15 @@ done
 command -v python >/dev/null 2>&1 || { printf 'ERROR: python is required\n' >&2; exit 1; }
 [[ -d "$RESULTS_ROOT/evaluation" ]] || { printf 'ERROR: no evaluation directory under %s\n' "$RESULTS_ROOT" >&2; exit 1; }
 
+label_args=()
+[[ -n "$RUN_LABEL" ]] && label_args=(--run-label "$RUN_LABEL")
+
 python "$SCRIPT_DIR/build-site-data.py" \
   --repo-root "$REPO_ROOT" \
   --results-root "$RESULTS_ROOT" \
   --suite "$SUITE" \
   --output "$OUTPUT" \
+  ${label_args[@]+"${label_args[@]}"} \
   --generated-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 
 python "$SCRIPT_DIR/render-page.py" --data "$OUTPUT" --output "${OUTPUT%%.json}.html"