increase max_steps in config and enhance system prompt; implement parallel execution for SWE-bench instances with improved logging and patch saving

Author: dcloud347

config.example.yaml

@@ -1,19 +1,71 @@
 agent:
-  max_steps: 30
+  max_steps: 80
   log_dir: logs/
   enable_memory: false
   system_prompt: |
-    You are a helpful assistant that can interact multiple times with a computer shell to solve programming tasks.
+    You are a software engineer that interacts multiple times with a computer shell to solve programming tasks.
     You operate in a REPL (Read-Eval-Print Loop) environment where you must issue exactly ONE shell tool call at a time.
     IMPORTANT: To execute commands you MUST call the shell tool — do NOT just write bash code blocks in your response text, as those are not executed.
     After each tool call, wait for the result before deciding the next step.
     Please briefly explain your reasoning before each tool call.
+
+    ## Recommended Workflow Phases (follow in order)
+    You have a limited number of steps. Spend them wisely across these phases:
+      1. **Explore** (≤12 steps): Understand the codebase and locate the relevant code.
+      2. **Reproduce** (≤5 steps): Build the project if needed and create a minimal reproduction app.
+      3. **Fix** (≤8 steps): Edit the source code to resolve the issue.
+      4. **Verify** (≤10 steps): Confirm the fix works using ask_web_agent and test edge cases.
+    If you are still exploring after 15 steps, stop and move to reproduction immediately. Do NOT spend too many steps exploring!
+
+    ## Important Boundaries
+      - DO NOT MODIFY: Tests, configuration files
+      - Only EXECUTE ONE command at a time, and wait for the result before issuing your next command.
+
+    ## Shell Usage Rules (CRITICAL)
+      - NEVER wrap commands in `bash -lc '...'` or `bash -c '...'`. You are already inside a bash session.
+      - When using grep/find, always search within /testbed (use `.` or `/testbed`). NEVER search from `..` or `/` as it will time out.
+      - Do NOT run blocking commands that take more than 60 seconds. Start servers/processes in the background (append `&`) and return immediately.
+
+    ## Efficient Code Reading
+      - Use `grep -n "pattern" -r src/ --include="*.js"` to find relevant lines first.
+      - Then read only the relevant section with `sed -n 'START,ENDp' file`. Keep ranges to ~100 lines maximum — this is a hard limit, not a suggestion.
+      - Do NOT use `cat` on source files. Do NOT use sed ranges larger than 100 lines. Outputs too long will be summarized and you will lose detail.
+      - Never search for something you already found — track what you know.
+
+    ## Editing Source Code
+      - Do NOT use `applypatch`, `patch`, or any tool not available in a standard bash environment.
+      - After each edit, verify the change took effect with `grep -n` or a small `sed -n` read.
+      - If an edit fails twice in a row, switch to a different approach immediately.
+    
+    ### Edit files with sed:
+      ```bash
+      # Replace all occurrences
+      sed -i 's/old_string/new_string/g' filename
+
+      # Replace only first occurrence
+      sed -i 's/old_string/new_string/' filename
+
+      # Replace first occurrence on line 1
+      sed -i '1s/old_string/new_string/' filename
+
+      # Replace all occurrences in lines 1-10
+      sed -i '1,10s/old_string/new_string/g' filename
+
+    ## Using ask_web_agent
+      - Call ask_web_agent at most 1-2 times per verification cycle.
+      - Each call should ask a specific, different question. Do NOT repeat the same question with different wording.
+      - If the result is ambiguous, act on your best interpretation rather than asking again.
+      - Once ask_web_agent confirms the fix is correct, stop verifying and conclude.
+
+    ## Tool Output
+      - If a tool result starts with "[Output was ... chars — summarized below]", the raw output was too long and has been automatically summarized for you. Treat the summary as the actual result and continue accordingly.
+
   LLM:
-    provider: openai     # "openai" | "anthropic"
+    provider: openai
     model: gpt-5
-    api_key: ""          # or set via LLM_API_KEY env var
-    base_url: ""         # leave empty for default endpoint
-    temperature: null    # null = use provider default; e.g. 0.0 for deterministic output
+    api_key: ""
+    base_url: ""
+    temperature: 0.5
   Memory:
     user_id: default
     base_url: http://localhost:1995/api/v1

run_swebench_multimodal.py

@@ -7,6 +7,8 @@
 import json
 import logging
 import random
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 from datasets import load_dataset
@@ -159,6 +161,32 @@ def _build_web_agent(cfg: WebAgentConfig, instance_id: str) -> WebAgent:
     )
 
 
+def _run_instance(cfg: Config, instance: dict, port: int, predictions_file, file_lock: threading.Lock) -> None:
+    """Run a single SWE-bench instance and write its prediction."""
+    instance_id = instance["instance_id"]
+    logger.info("=== %s starting (port %d) ===", instance_id, port)
+
+    web_agent = _build_web_agent(cfg.web_agent, instance_id)
+    agent = _build_agent(cfg.agent, instance, port, web_agent)
+    agent.run(_build_task_with_images(instance, port, cfg.agent.max_steps))
+
+    logger.info("=== %s finished, saving results ===", instance_id)
+    shell = next((t for t in agent.tool_list if isinstance(t, ShellTool)), None)
+    if shell is not None:
+        patch = shell.get_patch() or ""
+        record = json.dumps(
+            {
+                "instance_id": instance_id,
+                "model_name_or_path": "Argus",
+                "model_patch": patch,
+            }
+        )
+        with file_lock:
+            predictions_file.write(record + "\n")
+            predictions_file.flush()
+        logger.info("Patch written to predictions for %s", instance_id)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Run Argus on SWE-bench Multimodal")
     parser.add_argument("--config", default="config.yaml", help="Path to config.yaml")
@@ -169,6 +197,12 @@ def main() -> None:
         metavar="ID",
         help="Run only the specified instance IDs (runs all if omitted)",
     )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=1,
+        help="Number of parallel workers (default: 1)",
+    )
     args = parser.parse_args()
 
     cfg = Config.from_yaml(Path(__file__).parent / args.config)
@@ -178,41 +212,35 @@ def main() -> None:
         keep = set(args.instance_ids)
         dataset = dataset.filter(lambda x: x["instance_id"] in keep)
 
-    logger.info("Running %d instances", len(dataset))
+    logger.info("Running %d instances with %d worker(s)", len(dataset), args.workers)
 
     predictions_path = Path("predictions.jsonl")
+    file_lock = threading.Lock()
+
+    # Assign a unique port to each instance up front (thread-safe: done before threads start)
+    used_ports: set[int] = set()
+    instance_ports: list[int] = []
+    for _ in dataset:
+        port = random.randint(10000, 65535)
+        while port in used_ports:
+            port = random.randint(10000, 65535)
+        used_ports.add(port)
+        instance_ports.append(port)
 
-    used_port = []
     with predictions_path.open("a", encoding="utf-8") as predictions_file:
-        for instance in dataset:
-            instance_id = instance["instance_id"]
-            logger.info("=== %s ===", instance_id)
-
-            # Find an available port for this instance's server to run on.
-            port = random.randint(10000, 65535)
-            while port in used_port:
-                port = random.randint(10000, 65535)
-            used_port.append(port)
-
-            web_agent = _build_web_agent(cfg.web_agent, instance_id)
-            agent = _build_agent(cfg.agent, instance, port, web_agent)
-            agent.run(_build_task_with_images(instance, port, cfg.agent.max_steps))
-
-            # Generate and save git patch after agent finishes
-            logger.info("=== %s finished, saving results ===", instance_id)
-            shell = next((t for t in agent.tool_list if isinstance(t, ShellTool)), None)
-            if shell is not None:
-                patch = shell.get_patch() or ""
-                record = json.dumps(
-                    {
-                        "instance_id": instance_id,
-                        "model_name_or_path": "Argus",
-                        "model_patch": patch,
-                    }
-                )
-                predictions_file.write(record + "\n")
-                predictions_file.flush()
-                logger.info("Patch written to predictions for %s", instance_id)
+        with ThreadPoolExecutor(max_workers=args.workers) as executor:
+            futures = {
+                executor.submit(_run_instance, cfg, instance, port, predictions_file, file_lock): instance[
+                    "instance_id"
+                ]
+                for instance, port in zip(dataset, instance_ports)
+            }
+            for future in as_completed(futures):
+                instance_id = futures[future]
+                try:
+                    future.result()
+                except Exception:
+                    logger.exception("Instance %s failed", instance_id)
 
 
 if __name__ == "__main__":