Add Harbor runtime support and split the code to make it cleaner

Author: nancyjlau

docs/v6/advanced/harbor-convert.mdx

@@ -26,22 +26,29 @@ directly - one row per task dir (`id` = the dir name), sharing one declarative
 ```python
 from integrations.harbor import detect, load
 
-assert detect("./terminal-bench")
-taskset = load("./terminal-bench")
+assert detect("./harbor_tasks")
+taskset = load("./harbor_tasks")
 
 for task in taskset:
     print(task.env, task.id)
 ```
 
-Like every task row, the result carries no placement. Run it by supplying one -
-today that means a substrate already serving the control channel
-(`runtime=Runtime(url)`); a docker provider that builds and runs each task's
-`environment/` image is the planned follow-up:
+Like every task row, the result carries no placement. Run it by supplying one.
+For local Docker-backed Harbor execution, use `HarborRuntime`; it builds the
+task's `environment/` image, runs a fresh container, exposes the workspace
+through HUD's normal shell capability, and grades by running `tests/test.sh`:
 
 ```python
-from hud import Runtime
+from integrations.harbor import HarborRuntime
 
-job = await taskset.run(agent, runtime=Runtime("tcp://127.0.0.1:8765"))
+job = await taskset.run(agent, runtime=HarborRuntime("./harbor_tasks"))
+```
+
+The eval CLI also detects local Harbor task directories and datasets when using
+local runtime placement:
+
+```bash
+hud eval ./harbor_tasks claude --task-ids cancel-async-tasks --max-steps 30
 ```
 
 ## Export HUD tasks to Harbor

docs/v6/reference/cli.mdx

@@ -133,7 +133,8 @@ hud sync env                   # sync environment metadata
 ```
 
 External benchmark formats (currently Harbor) load directly into the runtime
-as `Taskset`s - no conversion step. See [Harbor interop](/v6/advanced/harbor-convert).
+as `Taskset`s - no conversion step. Local Harbor directories run with the Harbor
+Docker-backed runtime provider. See [Harbor interop](/v6/advanced/harbor-convert).
 
 ## Inspect
 

hud/cli/eval.py

@@ -728,7 +728,32 @@ def _spawn_target(source: Path) -> Path:
     return resolved.parent
 
 
-def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
+def _is_harbor_source(source_path: Path | None) -> bool:
+    if source_path is None or not source_path.exists():
+        return False
+    if not source_path.is_dir():
+        return False
+    from integrations.harbor import detect
+
+    return detect(source_path)
+
+
+def _load_local_taskset(source_path: Path) -> tuple[Any, str]:
+    from hud.eval import Taskset
+
+    if _is_harbor_source(source_path):
+        from integrations.harbor import load
+
+        return load(source_path), "harbor"
+    return Taskset.from_file(source_path), "hud"
+
+
+def _resolve_placement(
+    cfg: EvalConfig,
+    source_path: Path | None,
+    *,
+    source_kind: str = "hud",
+) -> Any:
     """Map the config's ``runtime`` onto a placement for ``Taskset.run``.
 
     "local" spawns each row's env from the source next to the tasks file;
@@ -744,6 +769,10 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
     if cfg.runtime == "local":
         if source_path is None:
             raise ValueError("local placement requires a local source path")
+        if source_kind == "harbor":
+            from integrations.harbor import HarborRuntime
+
+            return HarborRuntime(source_path)
         return LocalRuntime(_spawn_target(source_path))
     if cfg.runtime == "hud":
         require_api_key("run HUD runtime tunnel evals")
@@ -767,18 +796,19 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
     if cfg.source is None or cfg.agent_type is None:
         raise ValueError("source and agent_type must be set")
 
-    from hud.eval import Taskset
-
     source_path = Path(cfg.source)
     is_local = source_path.exists()
+    source_kind = "api"
     if is_local:
         hud_console.info(f"Loading tasks from: {cfg.source}")
         try:
-            taskset = Taskset.from_file(source_path)
+            taskset, source_kind = _load_local_taskset(source_path)
         except Exception as e:
             hud_console.error(f"Failed to load tasks from {cfg.source}: {e}")
             raise typer.Exit(1) from e
     else:
+        from hud.eval import Taskset
+
         hud_console.info(f"Loading platform taskset: {cfg.source}")
         try:
             taskset = Taskset.from_api(cfg.source)
@@ -832,7 +862,11 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
         )
 
     agent = _build_agent(cfg)
-    placement = _resolve_placement(cfg, source_path if is_local else None)
+    placement = _resolve_placement(
+        cfg,
+        source_path if is_local else None,
+        source_kind=source_kind,
+    )
 
     job = await taskset.run(
         agent,

hud/cli/tests/test_eval_config.py

@@ -20,6 +20,23 @@
 _ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/anthropic.claude"
 
 
+def _write_harbor_task(root: Path, name: str = "demo-task") -> Path:
+    task = root / name
+    (task / "environment").mkdir(parents=True)
+    (task / "tests").mkdir()
+    (task / "instruction.md").write_text("Fix the demo task.\n", encoding="utf-8")
+    (task / "task.toml").write_text(
+        'schema_version = "1.3"\n\n[task]\nname = "demo/demo-task"\n',
+        encoding="utf-8",
+    )
+    (task / "environment" / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8")
+    (task / "tests" / "test.sh").write_text(
+        "#!/usr/bin/env bash\nmkdir -p /logs/verifier\necho 1 > /logs/verifier/reward.txt\n",
+        encoding="utf-8",
+    )
+    return task
+
+
 def test_is_bedrock_arn() -> None:
     assert _is_bedrock_arn(_ARN) is True
     assert _is_bedrock_arn("claude-sonnet-4-6") is False
@@ -136,6 +153,30 @@ def test_resolve_placement_runtime_hud_uses_tunnel(
     assert isinstance(placement, HUDRuntime)
 
 
+def test_load_local_taskset_uses_harbor_loader_for_harbor_dirs(tmp_path: Path) -> None:
+    _write_harbor_task(tmp_path)
+
+    taskset, source_kind = eval_mod._load_local_taskset(tmp_path)
+
+    assert source_kind == "harbor"
+    assert len(taskset) == 1
+    assert taskset["demo-task"].id == "demo-task"
+
+
+def test_resolve_placement_local_harbor_uses_harbor_runtime(tmp_path: Path) -> None:
+    from integrations.harbor import HarborRuntime
+
+    _write_harbor_task(tmp_path)
+
+    placement = eval_mod._resolve_placement(
+        EvalConfig(runtime="local"),
+        tmp_path,
+        source_kind="harbor",
+    )
+
+    assert isinstance(placement, HarborRuntime)
+
+
 def test_resolve_placement_remote_uses_hosted_runtime(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

integrations/__init__.py

@@ -8,8 +8,9 @@
 This package lives outside ``hud`` on purpose: each module is a recipe built
 **only on the public SDK surface** (``Environment``, ``Task``,
 ``Taskset``, ``Runtime``) — that constraint is the proof the core is
-flexible. Copy a module into your project or run it from a checkout; nothing
-in the SDK or CLI imports it.
+flexible. Copy a module into your project or run it from a checkout. The CLI may
+call selected integrations explicitly for polished interop paths, but the
+integration contract itself stays independent of private SDK hooks.
 
 The contract: an integration module exposes ``detect(path) -> bool`` and
 ``load(path) -> Taskset``. Placement stays an execution-time concern — loaders

integrations/harbor.py

@@ -11,11 +11,9 @@
 
 :func:`load` parses a task dir (or a dataset of them) into rows sharing one
 env name per distinct ``environment/`` build context — no codegen, no
-roundtrip. Like every row, the result is runnable
-once a placement is supplied (``runtime=Runtime(url)`` against a served substrate
-today). Providers receive the row being placed, so a docker provider that
-builds and runs each row's ``environment/`` image is the named follow-up —
-expressible without engine changes.
+roundtrip. Like every row, the result is runnable once a placement is supplied.
+Use :class:`HarborRuntime` for local Docker-backed execution of Harbor tasks, or
+``runtime=Runtime(url)`` to attach to a substrate served elsewhere.
 
 :func:`export` is the reverse direction: turn a HUD task source into
 self-contained Harbor task folders (``task.toml`` + ``instruction.md`` +
@@ -40,19 +38,23 @@
 
 from __future__ import annotations
 
-import hashlib
 import json
 import logging
-import re
 import shutil
-import tomllib
-from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from hud.environment import Environment
 from hud.environment.server import TaskRunner
 from hud.eval import Task, Taskset
+from integrations.harbor_common import (
+    _HarborTask,
+    _is_harbor_task,
+    _parse_task,
+    _slugify,
+    _task_dirs,
+)
+from integrations.harbor_runtime import HarborRuntime
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -74,18 +76,12 @@
     "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache"
 )
 
-
 # ─── load: Harbor dirs -> Taskset ──────────────────────────────────────
 
 
 def detect(path: str | Path) -> bool:
     """True when *path* is a Harbor task dir or a dataset of them."""
-    root = Path(path)
-    if _is_harbor_task(root):
-        return True
-    if root.is_dir():
-        return any(_is_harbor_task(d) for d in root.iterdir() if d.is_dir())
-    return False
+    return bool(_task_dirs(path))
 
 
 def load(path: str | Path) -> Taskset:
@@ -96,12 +92,8 @@ def load(path: str | Path) -> Taskset:
     context (content-hashed), derived from the dataset name.
     """
     root = Path(path).resolve()
-    if _is_harbor_task(root):
-        task_dirs = [root]
-        dataset_name = root.parent.name
-    else:
-        task_dirs = sorted(d for d in root.iterdir() if d.is_dir() and _is_harbor_task(d))
-        dataset_name = root.name
+    task_dirs = _task_dirs(root)
+    dataset_name = root.parent.name if _is_harbor_task(root) else root.name
     if not task_dirs:
         raise ValueError(f"no Harbor tasks found in {path}")
 
@@ -126,54 +118,6 @@ def load(path: str | Path) -> Taskset:
     return Taskset(base_name, tasks)
 
 
-def _slugify(name: str) -> str:
-    """A valid env name (lowercase ``[a-z0-9-]``) from a dataset dir name."""
-    normalized = re.sub(r"[^a-z0-9-]", "", name.strip().lower().replace(" ", "-").replace("_", "-"))
-    return re.sub(r"-+", "-", normalized).strip("-") or "harbor"
-
-
-def _is_harbor_task(path: Path) -> bool:
-    return path.is_dir() and (path / "task.toml").exists() and (path / "instruction.md").exists()
-
-
-def _hash_directory(path: Path) -> str:
-    """Content-hash a directory for grouping tasks by identical environments."""
-    hasher = hashlib.sha256()
-    if not path.exists():
-        return "empty"
-    for file_path in sorted(path.rglob("*")):
-        if file_path.is_file():
-            hasher.update(str(file_path.relative_to(path)).encode())
-            hasher.update(file_path.read_bytes())
-    return hasher.hexdigest()[:16]
-
-
-@dataclass(frozen=True, slots=True)
-class _HarborTask:
-    """One parsed Harbor task dir."""
-
-    task_id: str
-    config: dict[str, Any]
-    env_hash: str
-
-
-def _parse_task(task_dir: Path) -> _HarborTask | None:
-    if not (task_dir / "instruction.md").is_file():
-        LOGGER.warning("failed to read instruction.md in %s", task_dir)
-        return None
-    try:
-        config: dict[str, Any] = tomllib.loads((task_dir / "task.toml").read_text("utf-8"))
-    except (OSError, tomllib.TOMLDecodeError):
-        LOGGER.warning("failed to parse task.toml in %s", task_dir)
-        config = {}
-    env_dir = task_dir / "environment"
-    return _HarborTask(
-        task_id=task_dir.name,
-        config=config,
-        env_hash=_hash_directory(env_dir) if env_dir.exists() else "no-env",
-    )
-
-
 # ─── export: HUD tasks -> Harbor task folders ───────────────────────────
 
 
@@ -443,6 +387,7 @@ async def export(
     "ALLOWED_PROTOCOLS",
     "CONTROL_PORT",
     "DEFAULT_ANSWER_FILE",
+    "HarborRuntime",
     "detect",
     "export",
     "load",