Refine Harbor runtime CLI integration

Author: nancyjlau

docs/v6/advanced/harbor-convert.mdx

@@ -9,8 +9,9 @@ task dirs - is a *frontend* that loads into the same primitives (`Environment`,
 `Task`, `Taskset`). Integrations are **loaders, not converters**: no codegen
 roundtrip to run foreign tasks. The Harbor integration lives in the SDK repo at
 [`integrations/harbor.py`](https://github.com/hud-evals/hud-python/blob/main/integrations/harbor.py)
-- a recipe built only on the public SDK surface; copy it into your project or
-run it from a checkout.
+- a public-surface loader that maps Harbor folders into SDK primitives. The
+included `HarborRuntime` is maintained with the SDK for local Docker execution;
+copy the loader into your project or run it from a checkout.
 
 ## Prerequisites
 
@@ -44,11 +45,11 @@ from integrations.harbor import HarborRuntime
 job = await taskset.run(agent, runtime=HarborRuntime("./harbor_tasks"))
 ```
 
-The eval CLI also detects local Harbor task directories and datasets when using
-local runtime placement:
+The eval CLI can run local Harbor task directories and datasets when you opt
+into the Harbor source format:
 
 ```bash
-hud eval ./harbor_tasks claude --task-ids cancel-async-tasks --max-steps 30
+hud eval ./harbor_tasks claude --format harbor --task-ids cancel-async-tasks --max-steps 30
 ```
 
 ## Export HUD tasks to Harbor

docs/v6/reference/cli.mdx

@@ -105,6 +105,7 @@ For a platform taskset, pass its name or id directly: `hud eval "My Tasks" claud
 | `--config`, `-c` | Agent config `key=value` (repeatable). |
 | `--verbose`, `-v` | Show agent logs (step progress, tool calls) for batch runs too. |
 | `--very-verbose`, `-vv` | Debug-level logs. |
+| `--format` | Task source format: `hud` (default) or `harbor`. |
 | `--runtime` | Placement: `local`, `hud` (HUD runtime tunnel), or `tcp://host:port`. Defaults to `local` for a tasks file; platform tasksets default to remote hosted execution. |
 | `--remote` | Run the whole rollout remotely on the HUD platform. |
 | `--yes`, `-y` | Skip confirmation prompt. |
@@ -133,8 +134,9 @@ hud sync env                   # sync environment metadata
 ```
 
 External benchmark formats (currently Harbor) load directly into the runtime
-as `Taskset`s - no conversion step. Local Harbor directories run with the Harbor
-Docker-backed runtime provider. See [Harbor interop](/v6/advanced/harbor-convert).
+as `Taskset`s - no conversion step. For local Harbor directories, opt in with
+`--format harbor` so the CLI uses the Harbor loader and Docker-backed runtime
+provider. See [Harbor interop](/v6/advanced/harbor-convert).
 
 ## Inspect
 

hud/cli/eval.py

@@ -63,6 +63,7 @@ def _resolve_model_from_catalog(model_id: str) -> tuple[AgentType, str] | None:
 
 _CONFIG_PATH = ".hud_eval.toml"
 _PLACEMENT_CONFLICT_ERROR = "--runtime and --remote are mutually exclusive placement options"
+_SOURCE_FORMATS = ("hud", "harbor")
 
 
 def _resolve_env_vars(obj: Any) -> Any:
@@ -167,6 +168,7 @@ class AgentPreset:
 # very_verbose = true
 # auto_respond = true
 # gateway = false  # Route LLM API calls through HUD Gateway
+# format = "hud"  # hud or harbor
 # runtime = "local"  # local, hud, or tcp://host:port
 # remote = false  # Run the whole rollout remotely on HUD
 
@@ -264,6 +266,7 @@ class EvalConfig(BaseModel):
         "group_size",
         "auto_respond",
         "gateway",
+        "format",
         "runtime",
         "remote",
     }
@@ -279,6 +282,9 @@ class EvalConfig(BaseModel):
     auto_respond: bool | None = None
     group_size: int = 1
     gateway: bool = False
+    #: Source format. ``None``/``hud`` means normal HUD task source loading;
+    #: ``harbor`` opts into the Harbor integration loader/runtime.
+    format: str | None = None
     #: Placement: "local" (spawn each row's env from the source), "hud"
     #: (HUD runtime tunnel), or a tcp:// url of an already-served env.
     #: ``None`` means "infer from the source": a local file runs locally, a
@@ -306,6 +312,20 @@ def _parse_agent_type(cls, v: Any) -> AgentType | None:
                 ) from None
         return v
 
+    @field_validator("format", mode="before")
+    @classmethod
+    def _parse_format(cls, v: Any) -> str | None:
+        if v is None:
+            return None
+        if not isinstance(v, str):
+            return v
+        normalized = v.strip().lower()
+        if normalized in ("", "hud"):
+            return None
+        if normalized in _SOURCE_FORMATS:
+            return normalized
+        raise ValueError(f"Invalid format: {v}. Must be one of: {', '.join(_SOURCE_FORMATS)}")
+
     def source_is_local_file(self) -> bool:
         """Whether ``source`` points at an on-disk taskset (vs. a platform slug/id)."""
         return self.source is not None and Path(self.source).exists()
@@ -319,6 +339,13 @@ def resolve_runtime(self) -> EvalConfig:
         ``--runtime`` is always honored, except ``local`` against a platform
         taskset, which has no env to spawn.
         """
+        if self.format == "harbor":
+            if not self.source_is_local_file():
+                hud_console.error("--format harbor requires a local Harbor task directory")
+                raise typer.Exit(1)
+            if self.remote or (self.runtime is not None and self.runtime != "local"):
+                hud_console.error("--format harbor currently supports only local runtime placement")
+                raise typer.Exit(1)
         if self.runtime is None:
             if self.source_is_local_file():
                 return self.model_copy(update={"runtime": "local"})
@@ -502,6 +529,7 @@ def merge_cli(
         gateway: bool = False,
         config: list[str] | None = None,
         task_ids: str | None = None,
+        format: str | None = None,
         runtime: str | None = None,
         remote: bool = False,
     ) -> EvalConfig:
@@ -517,6 +545,7 @@ def merge_cli(
                 "max_concurrent": max_concurrent,
                 "max_steps": max_steps,
                 "group_size": group_size,
+                "format": format,
                 "runtime": runtime,
             }.items()
             if value is not None
@@ -604,6 +633,8 @@ def display(self) -> None:
         table.add_column("Value", style="green")
 
         table.add_row("source", str(self.source or "-"))
+        if self.format:
+            table.add_row("format", self.format)
         table.add_row("runtime", str(self.runtime or "-"))
         table.add_row("agent", self.agent_type.value if self.agent_type else "-")
         if self.task_ids:
@@ -728,32 +759,20 @@ def _spawn_target(source: Path) -> Path:
     return resolved.parent
 
 
-def _is_harbor_source(source_path: Path | None) -> bool:
-    if source_path is None or not source_path.exists():
-        return False
-    if not source_path.is_dir():
-        return False
-    from integrations.harbor import detect
-
-    return detect(source_path)
-
-
-def _load_local_taskset(source_path: Path) -> tuple[Any, str]:
+def _load_local_taskset(source_path: Path, source_format: str | None) -> Any:
     from hud.eval import Taskset
 
-    if _is_harbor_source(source_path):
+    format_name = source_format or "hud"
+    if format_name == "hud":
+        return Taskset.from_file(source_path)
+    if format_name == "harbor":
         from integrations.harbor import load
 
-        return load(source_path), "harbor"
-    return Taskset.from_file(source_path), "hud"
+        return load(source_path)
+    raise ValueError(f"unsupported task source format: {format_name}")
 
 
-def _resolve_placement(
-    cfg: EvalConfig,
-    source_path: Path | None,
-    *,
-    source_kind: str = "hud",
-) -> Any:
+def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
     """Map the config's ``runtime`` onto a placement for ``Taskset.run``.
 
     "local" spawns each row's env from the source next to the tasks file;
@@ -769,7 +788,7 @@ def _resolve_placement(
     if cfg.runtime == "local":
         if source_path is None:
             raise ValueError("local placement requires a local source path")
-        if source_kind == "harbor":
+        if cfg.format == "harbor":
             from integrations.harbor import HarborRuntime
 
             return HarborRuntime(source_path)
@@ -798,11 +817,10 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
 
     source_path = Path(cfg.source)
     is_local = source_path.exists()
-    source_kind = "api"
     if is_local:
         hud_console.info(f"Loading tasks from: {cfg.source}")
         try:
-            taskset, source_kind = _load_local_taskset(source_path)
+            taskset = _load_local_taskset(source_path, cfg.format)
         except Exception as e:
             hud_console.error(f"Failed to load tasks from {cfg.source}: {e}")
             raise typer.Exit(1) from e
@@ -862,11 +880,7 @@ async def _run_evaluation(cfg: EvalConfig) -> Any:
         )
 
     agent = _build_agent(cfg)
-    placement = _resolve_placement(
-        cfg,
-        source_path if is_local else None,
-        source_kind=source_kind,
-    )
+    placement = _resolve_placement(cfg, source_path if is_local else None)
 
     job = await taskset.run(
         agent,
@@ -922,6 +936,11 @@ def eval_command(
     gateway: bool = typer.Option(
         False, "--gateway", "-g", help="Route LLM API calls through HUD Gateway"
     ),
+    format: str | None = typer.Option(
+        None,
+        "--format",
+        help="Task source format: hud (default) or harbor.",
+    ),
     runtime: str | None = typer.Option(
         None,
         "--runtime",
@@ -942,6 +961,7 @@ def eval_command(
         hud eval "My Tasks" claude-sonnet-4-6 --full   # Platform taskset, run on the platform
         hud eval tasks.json claude --config max_tokens=32768
         hud eval tasks.json claude --gateway           # Route LLM calls through HUD Gateway
+        hud eval ./harbor_tasks claude --format harbor # Run Harbor task dirs locally
         hud eval tasks.json claude-sonnet-4-6 --runtime hud  # Use HUD runtime tunnel
         hud eval tasks.json claude-sonnet-4-6 --remote       # Execute rollout remotely
     """
@@ -972,6 +992,7 @@ def eval_command(
             group_size=group_size,
             config=config,
             gateway=gateway,
+            format=format,
             runtime=runtime,
             remote=remote,
         )

hud/cli/tests/test_eval_config.py

@@ -153,30 +153,63 @@ def test_resolve_placement_runtime_hud_uses_tunnel(
     assert isinstance(placement, HUDRuntime)
 
 
-def test_load_local_taskset_uses_harbor_loader_for_harbor_dirs(tmp_path: Path) -> None:
+def test_load_local_taskset_uses_hud_loader_by_default(tmp_path: Path) -> None:
     _write_harbor_task(tmp_path)
 
-    taskset, source_kind = eval_mod._load_local_taskset(tmp_path)
+    taskset = eval_mod._load_local_taskset(tmp_path, None)
+
+    assert len(taskset) == 0
+
+
+def test_load_local_taskset_rejects_unknown_format(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match="unsupported task source format"):
+        eval_mod._load_local_taskset(tmp_path, "unknown")
+
+
+def test_load_local_taskset_uses_harbor_loader_when_format_is_harbor(tmp_path: Path) -> None:
+    _write_harbor_task(tmp_path)
+
+    taskset = eval_mod._load_local_taskset(tmp_path, "harbor")
 
-    assert source_kind == "harbor"
     assert len(taskset) == 1
     assert taskset["demo-task"].id == "demo-task"
 
 
-def test_resolve_placement_local_harbor_uses_harbor_runtime(tmp_path: Path) -> None:
+def test_resolve_placement_local_harbor_format_uses_harbor_runtime(tmp_path: Path) -> None:
     from integrations.harbor import HarborRuntime
 
     _write_harbor_task(tmp_path)
 
     placement = eval_mod._resolve_placement(
-        EvalConfig(runtime="local"),
+        EvalConfig(runtime="local", format="harbor"),
         tmp_path,
-        source_kind="harbor",
     )
 
     assert isinstance(placement, HarborRuntime)
 
 
+def test_resolve_placement_local_hud_format_uses_local_runtime(tmp_path: Path) -> None:
+    from hud.eval import LocalRuntime
+
+    _write_harbor_task(tmp_path)
+
+    placement = eval_mod._resolve_placement(EvalConfig(runtime="local"), tmp_path)
+
+    assert isinstance(placement, LocalRuntime)
+
+
+def test_harbor_format_rejects_nonlocal_source() -> None:
+    with pytest.raises(typer.Exit):
+        EvalConfig(source="platform/taskset", format="harbor").resolve_runtime()
+
+
+def test_harbor_format_rejects_nonlocal_runtime(tmp_path: Path) -> None:
+    _write_harbor_task(tmp_path)
+
+    with pytest.raises(typer.Exit):
+        EvalConfig(source=str(tmp_path), format="harbor", runtime="hud").resolve_runtime()
+
+
 def test_resolve_placement_remote_uses_hosted_runtime(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

integrations/__init__.py

@@ -5,12 +5,12 @@
 primitives. Integrations are **loaders, not converters**: no codegen roundtrip
 to run foreign tasks.
 
-This package lives outside ``hud`` on purpose: each module is a recipe built
-**only on the public SDK surface** (``Environment``, ``Task``,
-``Taskset``, ``Runtime``) — that constraint is the proof the core is
-flexible. Copy a module into your project or run it from a checkout. The CLI may
-call selected integrations explicitly for polished interop paths, but the
-integration contract itself stays independent of private SDK hooks.
+This package lives outside ``hud`` on purpose: loaders are recipes built on the
+public SDK surface (``Environment``, ``Task``, ``Taskset``, ``Runtime``). Copy a
+loader into your project or run it from a checkout. The CLI may call selected
+integrations explicitly for polished interop paths. A repo-maintained
+integration may also expose a local provider for that explicit CLI path; that
+provider is SDK implementation code, not the portable loader contract.
 
 The contract: an integration module exposes ``detect(path) -> bool`` and
 ``load(path) -> Taskset``. Placement stays an execution-time concern — loaders