Merge pull request #477 from hud-evals/fix-template-annotation-resolution

feat: enhance parameter annotation resolution in task signatures

Author: jdchawla29

hud/environment/env.py

@@ -98,7 +98,7 @@ def __init__(
         #: Type the agent must produce (``None`` = plain text). Drives answer
         #: deserialization into ``Answer[T]``.
         self.return_type = returns
-        self.sig = inspect.signature(func)
+        self.sig = inspect.signature(func, eval_str=True)
         functools.update_wrapper(self, func)
 
     def manifest_entry(self) -> dict[str, Any]:

hud/environment/tests/test_manifest.py

@@ -7,6 +7,8 @@
 
 from __future__ import annotations
 
+from typing import Literal
+
 from pydantic import BaseModel
 
 from hud.environment import Environment
@@ -17,6 +19,14 @@ class _Point(BaseModel):
     y: int
 
 
+class _Payload(BaseModel):
+    text: str
+
+
+_Mode = Literal["easy", "hard"]
+_Retries = int | None
+
+
 def test_args_schema_captures_params_defaults_and_required() -> None:
     env = Environment("manifests")
 
@@ -86,3 +96,19 @@ async def typed():
     assert entry["input"]["properties"]["x"]["type"] == "integer"
     assert entry["returns"]["properties"]["y"]["type"] == "integer"
     assert entry["args"]["properties"] == {}
+
+
+def test_args_schema_resolves_postponed_rich_annotations() -> None:
+    env = Environment("manifests")
+
+    @env.template()
+    async def rich(mode: _Mode, payload: _Payload, retries: _Retries = None):
+        yield "go"
+        yield 1.0
+
+    assert callable(rich)
+    schema = env.tasks["rich"].manifest_entry()["args"]
+    assert schema["properties"]["mode"]["enum"] == ["easy", "hard"]
+    assert schema["properties"]["retries"]["default"] is None
+    assert "$ref" in schema["properties"]["payload"]
+    assert set(schema["required"]) == {"mode", "payload"}

hud/environment/tests/test_server.py

@@ -7,7 +7,10 @@
 
 from __future__ import annotations
 
+from typing import Literal
+
 import pytest
+from pydantic import BaseModel
 
 from hud.clients import HudProtocolError
 from hud.environment import Answer, Environment
@@ -17,6 +20,13 @@
 from .conftest import served
 
 
+class _Payload(BaseModel):
+    text: str
+
+
+_Mode = Literal["upper", "lower"]
+
+
 async def test_dict_grade_without_numeric_score_errors_loudly() -> None:
     env = Environment("badgrade")
 
@@ -79,3 +89,34 @@ def test_answer_holds_parsed_content_and_raw_string() -> None:
     answer = Answer(content={"final": "42"}, raw='{"final": "42"}')
     assert answer.content == {"final": "42"}
     assert answer.raw == '{"final": "42"}'
+
+
+async def test_start_coerces_postponed_rich_annotations() -> None:
+    env = Environment("coerce")
+
+    @env.template()
+    async def typed(mode: _Mode, payload: _Payload, retries: int | None = None):
+        if mode == "upper":
+            prompt = payload.text.upper()
+        elif mode == "lower":
+            prompt = payload.text.lower()
+        else:
+            raise ValueError(f"unexpected mode: {mode!r}")
+        if retries is not None:
+            prompt += "!" * retries
+        yield prompt
+        yield 1.0
+
+    assert callable(typed)
+    async with served(env) as client:
+        async with Run(
+            client,
+            "typed",
+            {
+                "mode": '"upper"',
+                "payload": '{"text":"hello"}',
+                "retries": "3",
+            },
+        ) as run:
+            run.trace.content = "x"
+        assert run.prompt == "HELLO!!!"