Version to 1.14.0

Author: monoxgas

docs/sdk/airt.mdx

@@ -754,8 +754,8 @@ def hop_skip_jump_search(  # noqa: PLR0915
             # 3d - Projection
 
             projector = bisection_image_search(
-                start_image=source,
-                end_image=current,
+                source,
+                current,
                 decision_objective=decision_objective,
                 decision_threshold=decision_threshold,
                 tolerance=theta,

docs/sdk/api.mdx

@@ -1004,9 +1004,9 @@ def poll_for_token(
             "POST", "/auth/device/token", json_data={"device_code": device_code}
         )
 
-        if response.status_code == 200:  # noqa: PLR2004
+        if response.status_code == 200:
             return AccessRefreshTokenResponse(**response.json())
-        if response.status_code != 401:  # noqa: PLR2004
+        if response.status_code != 401:
             raise RuntimeError(self._get_error_message(response))
 
         time.sleep(interval)

docs/sdk/transforms.mdx

@@ -490,7 +490,7 @@ Encodes text using the Caesar cipher.
 def caesar_cipher(offset: int, *, name: str = "caesar") -> Transform[str, str]:
     """Encodes text using the Caesar cipher."""
 
-    if not -25 <= offset <= 25:  # noqa: PLR2004
+    if not -25 <= offset <= 25:
         raise ValueError("Caesar offset must be between -25 and 25.")
 
     def transform(
@@ -551,9 +551,9 @@ def rot47_cipher(*, name: str = "rot47") -> Transform[str, str]:
         transformed = []
         for char in text:
             char_ord = ord(char)
-            if 33 <= char_ord <= 126:  # noqa: PLR2004
+            if 33 <= char_ord <= 126:
                 shifted_ord = char_ord + 47
-                if shifted_ord > 126:  # noqa: PLR2004
+                if shifted_ord > 126:
                     shifted_ord -= 94
                 transformed.append(chr(shifted_ord))
             else:
@@ -1502,7 +1502,7 @@ def zalgo(
         seed: Random seed for reproducibility.
         name: Name of the transform.
     """
-    if not 0 <= intensity <= 100:  # noqa: PLR2004
+    if not 0 <= intensity <= 100:
         raise ValueError("Intensity must be between 0 and 100.")
     if not 0.0 <= ratio <= 1.0:
         raise ValueError("Application ratio must be between 0.0 and 1.0.")
@@ -2914,7 +2914,7 @@ def swap(
         ),
     ) -> str:
         items = list(text) if unit == "char" else re.findall(r"\w+|\S+", text)
-        if len(items) < 2:  # noqa: PLR2004
+        if len(items) < 2:
             return text
 
         num_to_swap = int(len(items) * ratio)

docs/usage/evals.mdx

@@ -256,4 +256,123 @@ When you run this code, you will see a TUI that includes:
 -   **Event Log:** A timestamped log of key events, such as a sample failing.
 -   **Summary Statistics:** A live-updating summary of the pass/fail rate.
 
-Using `.console()` is perfect for interactive development and for monitoring large benchmark runs.
+Using `.console()` is perfect for interactive development and for monitoring large benchmark runs.
+
+### **Advanced Patterns**
+
+Once you are comfortable with the basics, you can use these advanced features to build more resilient and sophisticated evaluation pipelines.
+
+#### **Loading Datasets from Files**
+
+For larger evaluations, defining your dataset in-memory isn't practical. You can load a dataset directly from a file by providing a string or `pathlib.Path` object. Supported formats include `.jsonl`, `.csv`, `.json`, and `.yaml`.
+
+Let's assume you have a file named `dataset.jsonl` with the following content:
+
+```json title="dataset.jsonl"
+{"country": "France", "expected_capital": "Paris"}
+{"country": "Japan", "expected_capital": "Tokyo"}
+```
+
+You can then reference this file directly in your `Eval`.
+
+```python
+import dreadnode as dn
+from dreadnode import scorers
+
+@dn.task
+async def get_capital(country: str) -> str:
+    capitals = {"France": "Paris", "Japan": "Tokyo"}
+    return capitals.get(country, "I don't know.")
+
+correctness_check = scorers.equals(dn.DatasetField("expected_capital")) >> "is_correct"
+
+# Simply pass the file path to the `dataset` argument.
+file_based_eval = get_capital.as_eval(
+    dataset="dataset.jsonl",
+    dataset_input_mapping=["country"],
+    scorers=[correctness_check],
+    assert_scores=["is_correct"],
+)
+
+result = await file_based_eval.run()
+print(f"Pass Rate: {result.pass_rate:.2%}")
+```
+
+#### **Customizing Input Mapping**
+
+The system can automatically map dataset columns to task parameters if their names match. However, if your dataset columns have different names than your task's parameters, you must provide an explicit mapping using `dataset_input_mapping`.
+
+Here's how you would map a dataset with a `location` column to the task's `country` parameter.
+
+```python
+import dreadnode as dn
+from dreadnode import scorers
+
+@dn.task
+async def get_capital(country: str) -> str: # Task expects `country`
+    capitals = {"France": "Paris"}
+    return capitals.get(country, "I don't know.")
+
+# Dataset uses `location` instead of `country`.
+dataset_with_mismatched_keys = [
+    {"location": "France", "expected_capital": "Paris"}
+]
+
+# Use a dict to map `dataset_key: task_parameter_name`.
+mapping_eval = get_capital.as_eval(
+    dataset=dataset_with_mismatched_keys,
+    dataset_input_mapping={"location": "country"},
+    scorers=[scorers.equals(dn.DatasetField("expected_capital")) >> "is_correct"],
+)
+
+result = await mapping_eval.run()
+print(result.samples[0].input)
+```
+
+#### **Building Resilient Evaluations**
+
+When working with large datasets or non-deterministic tasks, some samples may fail due to transient issues or bad data. You can configure your `Eval` to tolerate a certain number of failures without stopping the entire run.
+
+-   `max_errors`: The total number of sample errors to tolerate before stopping.
+-   `max_consecutive_errors`: The number of *consecutive* sample errors to tolerate before stopping.
+
+```python
+@dn.task
+async def flaky_task(value: int) -> int:
+    if value == 2:
+        raise ValueError("This value causes an error!")
+    return value * 2
+
+dataset = [{"value": 1}, {"value": 2}, {"value": 3}, {"value": 4}]
+
+# This evaluation will stop after the first error.
+# result = await flaky_task.as_eval(dataset=dataset).run() # This would raise an error and stop.
+
+# This evaluation will tolerate up to 5 total errors and continue running.
+resilient_eval = flaky_task.as_eval(
+    dataset=dataset,
+    max_errors=5,
+)
+
+result = await resilient_eval.run()
+print(f"Total samples processed: {len(result.samples)}")
+print(f"Samples with errors: {len([s for s in result.samples if s.error])}")
+```
+
+#### **Programmatic Event Streaming**
+
+The `.console()` method is a convenient wrapper around a lower-level event stream. If you need to build custom logic or UIs based on evaluation events, you can consume this stream directly using `async with eval.stream()`.
+
+This is useful for advanced cases like sending real-time alerts or implementing custom early-stopping logic.
+
+```python
+# Uses the `capital_eval` from a previous example.
+async with capital_eval.stream() as stream:
+    async for event in stream:
+        if isinstance(event, dn.eval.SampleComplete):
+            if event.sample.failed:
+                print(f"Detected a failure on sample {event.sample.index}!")
+        elif isinstance(event, dn.eval.EvalEnd):
+            print(f"Evaluation finished with stop reason: {event.stop_reason}")
+
+```

dreadnode/agent/hooks/summarize.py

@@ -66,7 +66,7 @@ def summarize_when_long(
         min_messages_to_keep: The minimum number of messages to retain after summarization (default is 5).
     """
 
-    if min_messages_to_keep < 2:  # noqa: PLR2004
+    if min_messages_to_keep < 2:
         raise ValueError("min_messages_to_keep must be at least 2.")
 
     @component

dreadnode/agent/stop.py

@@ -318,7 +318,7 @@ def elapsed_time(max_seconds: int) -> StopCondition:
     """
 
     def stop(events: Sequence[AgentEvent]) -> bool:
-        if len(events) < 2:  # noqa: PLR2004
+        if len(events) < 2:
             return False
 
         first_event = events[0]

dreadnode/airt/search/hop_skip_jump.py

@@ -169,8 +169,8 @@ def is_adversarial(trial: Trial) -> bool:
             # 3d - Projection
 
             projector = bisection_image_search(
-                start_image=source,
-                end_image=current,
+                source,
+                current,
                 decision_objective=decision_objective,
                 decision_threshold=decision_threshold,
                 tolerance=theta,

dreadnode/api/client.py

@@ -241,9 +241,9 @@ def poll_for_token(
                 "POST", "/auth/device/token", json_data={"device_code": device_code}
             )
 
-            if response.status_code == 200:  # noqa: PLR2004
+            if response.status_code == 200:
                 return AccessRefreshTokenResponse(**response.json())
-            if response.status_code != 401:  # noqa: PLR2004
+            if response.status_code != 401:
                 raise RuntimeError(self._get_error_message(response))
 
             time.sleep(interval)
@@ -591,10 +591,10 @@ def _write_chunk_file(
             # Single run - use the run ID
             run_id = df["run_id"].iloc[0]
             base_name = f"run_{run_id}"
-        elif total_runs <= 10:  # noqa: PLR2004
+        elif total_runs <= 10:
             # Few runs - include count
             base_name = f"runs_{total_runs}_page_{page}"
-        elif total_runs <= 100:  # noqa: PLR2004
+        elif total_runs <= 100:
             # Medium batch - include count
             base_name = f"runs_{total_runs}_batch_{page}"
         else:

dreadnode/cli/github.py

@@ -146,7 +146,7 @@ def tree_url(self) -> str:
     def exists(self) -> bool:
         """Check if a repo exists (or is private) on GitHub."""
         response = httpx.get(f"https://github.com/{self.namespace}/{self.repo}")
-        return response.status_code == 200  # noqa: PLR2004
+        return response.status_code == 200
 
     def __repr__(self) -> str:
         return f"GithubRepo(namespace='{self.namespace}', repo='{self.repo}', ref='{self.ref}')"

dreadnode/eval/eval.py

@@ -149,7 +149,7 @@ def __repr__(self) -> str:
     def _generic_types(cls) -> tuple[type[In], type[Out]]:
         for c in cls.__mro__:
             metadata = getattr(c, "__pydantic_generic_metadata__", {})
-            if len(args := (metadata.get("args", ()) or getattr(c, "__args__", ()))) == 2:  # noqa: PLR2004
+            if len(args := (metadata.get("args", ()) or getattr(c, "__args__", ()))) == 2:
                 return args  # type: ignore[no-any-return]
         return t.Any, t.Any  # type: ignore[return-value]