From e8cd13045e30152898c29bde5f2d890d84d1c6be Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Mon, 6 Jul 2026 09:09:06 +0000 Subject: [PATCH 1/9] fix: fixed the blocking fire and forget issue. - Wrapped all functions (`notify_call_start`, `notify_call_complete` and `notify_call_error`) into `asyncio.Task` so parent function don't have to await them. - Converted the `_emit_execution_transition_log` to sync as underlying code is sync and execute it to a separate thread. --- sdk/python/agentfield/agent.py | 180 +++++++++++++++--------- sdk/python/agentfield/agent_workflow.py | 20 ++- 2 files changed, 125 insertions(+), 75 deletions(-) diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 5f120ea61..eca307de3 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -2245,13 +2245,17 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: execution_context.reasoner_name = reasoner_id - await self.workflow_handler.notify_call_start( - execution_context.execution_id, - execution_context, - reasoner_id, - payload_dict, - parent_execution_id=execution_context.parent_execution_id, + task = asyncio.create_task( + self.workflow_handler.notify_call_start( + execution_context.execution_id, + execution_context, + reasoner_id, + payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) start_time = time.time() @@ -2358,29 +2362,38 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - await self.workflow_handler.notify_call_complete( - execution_context.execution_id, - execution_context.workflow_id, - result, - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + task = asyncio.create_task( + self.workflow_handler.notify_call_complete( + execution_context.execution_id, + execution_context.workflow_id, + result, + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) return result except asyncio.CancelledError as cancel_err: if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - await self.workflow_handler.notify_call_error( - execution_context.execution_id, - execution_context.workflow_id, - "Execution cancelled by upstream client", - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_error( + execution_context.execution_id, + execution_context.workflow_id, + "Execution cancelled by upstream client", + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) raise cancel_err except ExecuteError as exec_err: # Propagate upstream HTTP status codes from cross-agent calls. @@ -2388,15 +2401,20 @@ async def _execute_reasoner_endpoint( # (unhandled exception) and then 502 at the outer control plane. if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - await self.workflow_handler.notify_call_error( - execution_context.execution_id, - execution_context.workflow_id, - str(exec_err), - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_error( + execution_context.execution_id, + execution_context.workflow_id, + str(exec_err), + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) detail = {"error": str(exec_err)} if exec_err.error_details: detail["error_details"] = exec_err.error_details @@ -2408,28 +2426,38 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() detail = getattr(http_exc, "detail", None) or str(http_exc) - await self.workflow_handler.notify_call_error( - execution_context.execution_id, - execution_context.workflow_id, - detail, - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_error( + execution_context.execution_id, + execution_context.workflow_id, + detail, + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) raise except Exception as exc: if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - await self.workflow_handler.notify_call_error( - execution_context.execution_id, - execution_context.workflow_id, - str(exc), - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_error( + execution_context.execution_id, + execution_context.workflow_id, + str(exc), + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) raise finally: reset_execution_context(context_token) @@ -3110,40 +3138,54 @@ async def _run_async_skill(*args, **kwargs): previous_ctx = self._current_execution_context self._current_execution_context = child_context input_payload = _build_invocation_payload(args, kwargs) - - await self.workflow_handler.notify_call_start( - child_context.execution_id, - child_context, - skill_id, - input_payload, - parent_execution_id=current_context.execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_start( + child_context.execution_id, + child_context, + skill_id, + input_payload, + parent_execution_id=current_context.execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) start_time = time.time() try: result = await original_func(*args, **kwargs) duration_ms = int((time.time() - start_time) * 1000) - await self.workflow_handler.notify_call_complete( - child_context.execution_id, - child_context.workflow_id, - result, - duration_ms, - child_context, - input_data=input_payload, - parent_execution_id=current_context.execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_complete( + child_context.execution_id, + child_context.workflow_id, + result, + duration_ms, + child_context, + input_data=input_payload, + parent_execution_id=current_context.execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) return result except Exception as exc: duration_ms = int((time.time() - start_time) * 1000) - await self.workflow_handler.notify_call_error( - child_context.execution_id, - child_context.workflow_id, - str(exc), - duration_ms, - child_context, - input_data=input_payload, - parent_execution_id=current_context.execution_id, + + task = asyncio.create_task( + self.workflow_handler.notify_call_error( + child_context.execution_id, + child_context.workflow_id, + str(exc), + duration_ms, + child_context, + input_data=input_payload, + parent_execution_id=current_context.execution_id, + ) ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) raise finally: reset_execution_context(token) diff --git a/sdk/python/agentfield/agent_workflow.py b/sdk/python/agentfield/agent_workflow.py index f99efcdec..37c663fed 100644 --- a/sdk/python/agentfield/agent_workflow.py +++ b/sdk/python/agentfield/agent_workflow.py @@ -1,3 +1,4 @@ +import asyncio import inspect import time from typing import Any, Callable, Dict, Optional @@ -121,7 +122,8 @@ async def notify_call_start( *, parent_execution_id: Optional[str] = None, ) -> None: - await self._emit_execution_transition_log( + await asyncio.to_thread( + self._emit_execution_transition_log, context, reasoner_name, event_type="reasoner.started", @@ -131,6 +133,7 @@ async def notify_call_start( input_data=input_data, parent_execution_id=parent_execution_id, ) + payload = self._build_event_payload( context, reasoner_name, @@ -151,7 +154,8 @@ async def notify_call_complete( input_data: Optional[Dict[str, Any]] = None, parent_execution_id: Optional[str] = None, ) -> None: - await self._emit_execution_transition_log( + await asyncio.to_thread( + self._emit_execution_transition_log, context, context.reasoner_name, event_type="reasoner.completed", @@ -163,6 +167,7 @@ async def notify_call_complete( input_data=input_data, parent_execution_id=parent_execution_id, ) + payload = self._build_event_payload( context, context.reasoner_name, @@ -185,7 +190,8 @@ async def notify_call_error( input_data: Optional[Dict[str, Any]] = None, parent_execution_id: Optional[str] = None, ) -> None: - await self._emit_execution_transition_log( + await asyncio.to_thread( + self._emit_execution_transition_log, context, context.reasoner_name, event_type="reasoner.failed", @@ -291,7 +297,8 @@ async def _ensure_execution_registered( context.execution_id = body.get("execution_id", context.execution_id) context.workflow_id = body.get("workflow_id", context.workflow_id) context.run_id = body.get("run_id", context.run_id) - await self._emit_execution_transition_log( + await asyncio.to_thread( + self._emit_execution_transition_log, context, reasoner_name, event_type="execution.registered", @@ -303,7 +310,8 @@ async def _ensure_execution_registered( except Exception as exc: # pragma: no cover - network failure path if getattr(self.agent, "dev_mode", False): log_warn(f"Workflow registration failed: {exc}") - await self._emit_execution_transition_log( + await asyncio.to_thread( + self._emit_execution_transition_log, context, reasoner_name, event_type="execution.registration.failed", @@ -348,7 +356,7 @@ def _build_event_payload( payload["input_data"] = input_data return payload - async def _emit_execution_transition_log( + def _emit_execution_transition_log( self, context: ExecutionContext, reasoner_name: str, From cc248e6a4878efea78c0faaca7b3a799785f5353 Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Mon, 6 Jul 2026 10:39:47 +0000 Subject: [PATCH 2/9] revert: I reverted the execution of `_emit_execution_transition_log` method using `asyncio.to_thread()` because of mismatch order. - Considering the `log_execution` inside the function is a lightweight function which don't takeover the `event_loop` for long time. --- sdk/python/agentfield/agent_workflow.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/sdk/python/agentfield/agent_workflow.py b/sdk/python/agentfield/agent_workflow.py index 37c663fed..fb5d833cf 100644 --- a/sdk/python/agentfield/agent_workflow.py +++ b/sdk/python/agentfield/agent_workflow.py @@ -122,8 +122,7 @@ async def notify_call_start( *, parent_execution_id: Optional[str] = None, ) -> None: - await asyncio.to_thread( - self._emit_execution_transition_log, + self._emit_execution_transition_log( context, reasoner_name, event_type="reasoner.started", @@ -154,8 +153,7 @@ async def notify_call_complete( input_data: Optional[Dict[str, Any]] = None, parent_execution_id: Optional[str] = None, ) -> None: - await asyncio.to_thread( - self._emit_execution_transition_log, + self._emit_execution_transition_log( context, context.reasoner_name, event_type="reasoner.completed", @@ -190,8 +188,7 @@ async def notify_call_error( input_data: Optional[Dict[str, Any]] = None, parent_execution_id: Optional[str] = None, ) -> None: - await asyncio.to_thread( - self._emit_execution_transition_log, + self._emit_execution_transition_log( context, context.reasoner_name, event_type="reasoner.failed", @@ -297,8 +294,7 @@ async def _ensure_execution_registered( context.execution_id = body.get("execution_id", context.execution_id) context.workflow_id = body.get("workflow_id", context.workflow_id) context.run_id = body.get("run_id", context.run_id) - await asyncio.to_thread( - self._emit_execution_transition_log, + self._emit_execution_transition_log( context, reasoner_name, event_type="execution.registered", @@ -310,8 +306,7 @@ async def _ensure_execution_registered( except Exception as exc: # pragma: no cover - network failure path if getattr(self.agent, "dev_mode", False): log_warn(f"Workflow registration failed: {exc}") - await asyncio.to_thread( - self._emit_execution_transition_log, + self._emit_execution_transition_log( context, reasoner_name, event_type="execution.registration.failed", From 7b67b380c1d101018185beee57d96ef923f73366 Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Mon, 6 Jul 2026 13:40:31 +0000 Subject: [PATCH 3/9] lint: Removed unused import --- sdk/python/agentfield/agent_workflow.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/agentfield/agent_workflow.py b/sdk/python/agentfield/agent_workflow.py index fb5d833cf..ac6222132 100644 --- a/sdk/python/agentfield/agent_workflow.py +++ b/sdk/python/agentfield/agent_workflow.py @@ -1,4 +1,3 @@ -import asyncio import inspect import time from typing import Any, Callable, Dict, Optional From 948eb2969459ef455c295c74832d9ec659e90f3b Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Sat, 11 Jul 2026 04:26:57 +0000 Subject: [PATCH 4/9] fix: Added missing background task cleanup in `_cleanup_async_resources` method. --- sdk/python/agentfield/agent.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index eca307de3..ad8f5bf80 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -4415,6 +4415,20 @@ async def _cleanup_async_resources(self) -> None: if self.dev_mode: log_debug(f"Error cleaning up AsyncExecutionManager: {e}") + if self._background_tasks: + try: + await asyncio.wait_for( + asyncio.gather(*self._background_tasks, return_exceptions=True), + timeout = 5 + ) + if self.dev_mode: + log_debug("Background tasks are cleaned up") + except Exception as e: + if self.dev_mode: + log_debug(f"Error cleaning up background tasks: {e}") + finally: + self._background_tasks.clear() + if getattr(self, "client", None) is not None: try: await self.client.aclose() From 40b83c2d38b32045d2aeaa002408d4e11d392b63 Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Sat, 11 Jul 2026 05:00:37 +0000 Subject: [PATCH 5/9] lint: Used ruff format --- sdk/python/agentfield/agent.py | 105 +++++--- sdk/python/agentfield/agent_discovery.py | 10 +- sdk/python/agentfield/agent_field_handler.py | 16 +- sdk/python/agentfield/agent_schema.py | 5 +- sdk/python/agentfield/agent_serverless.py | 3 +- sdk/python/agentfield/agent_vc.py | 15 +- .../agentfield/async_execution_manager.py | 46 ++-- sdk/python/agentfield/client.py | 105 +++++--- sdk/python/agentfield/decorator_metadata.py | 4 +- sdk/python/agentfield/did_auth.py | 10 +- sdk/python/agentfield/did_manager.py | 5 +- sdk/python/agentfield/exceptions.py | 4 +- sdk/python/agentfield/execution_state.py | 10 +- sdk/python/agentfield/harness/_cli.py | 4 +- .../agentfield/http_connection_manager.py | 4 +- sdk/python/agentfield/media_providers.py | 19 +- sdk/python/agentfield/memory.py | 19 +- sdk/python/agentfield/memory_events.py | 12 +- sdk/python/agentfield/router.py | 2 +- sdk/python/agentfield/session_transport.py | 20 +- sdk/python/agentfield/tool_calling.py | 4 +- sdk/python/agentfield/verification.py | 50 +++- .../tests/functional/test_agent_execution.py | 24 +- .../functional/test_agent_registration.py | 24 +- .../functional/test_agent_to_agent_call.py | 24 +- .../tests/functional/test_ai_integration.py | 12 +- sdk/python/tests/helpers.py | 8 +- sdk/python/tests/integration/conftest.py | 6 +- .../tests/mypy_reveal_simulate_schedule.py | 1 + .../tests/mypy_reveal_simulate_trigger.py | 1 + sdk/python/tests/test_accepts_webhook.py | 6 +- .../tests/test_agent_ai_coverage_additions.py | 12 +- .../tests/test_agent_ai_deadlock_recovery.py | 41 ++- .../tests/test_agent_bigfiles_final90.py | 21 +- .../tests/test_agent_coverage_additions.py | 8 +- sdk/python/tests/test_agent_integration.py | 6 +- .../tests/test_agent_lifecycle_invariants.py | 10 +- sdk/python/tests/test_agent_networking.py | 17 +- sdk/python/tests/test_agent_registry.py | 1 + sdk/python/tests/test_agent_server.py | 43 ++-- .../tests/test_agent_server_extended.py | 4 +- sdk/python/tests/test_agent_serverless.py | 107 ++++---- sdk/python/tests/test_agent_workflow.py | 6 +- .../tests/test_agent_workflow_extended.py | 12 +- sdk/python/tests/test_approval.py | 60 +++-- sdk/python/tests/test_async_execution.py | 7 +- .../test_async_execution_manager_final90.py | 36 ++- .../test_async_execution_manager_paths.py | 1 + sdk/python/tests/test_client.py | 6 + sdk/python/tests/test_client_auth.py | 18 +- .../tests/test_client_bigfiles_coverage.py | 34 ++- .../tests/test_client_coverage_additions.py | 24 +- .../tests/test_client_execution_vc_payload.py | 4 +- sdk/python/tests/test_client_laser_push.py | 240 +++++++++++++----- sdk/python/tests/test_client_unit.py | 4 +- sdk/python/tests/test_connection_manager.py | 9 +- .../test_connection_manager_invariants.py | 27 +- sdk/python/tests/test_crypto.py | 12 +- .../tests/test_decorator_code_origin.py | 8 +- sdk/python/tests/test_did_auth.py | 18 +- sdk/python/tests/test_did_auth_invariants.py | 39 ++- sdk/python/tests/test_exceptions.py | 40 +-- .../tests/test_execution_context_parent_vc.py | 28 +- sdk/python/tests/test_execution_logger.py | 23 +- .../tests/test_execution_state_invariants.py | 66 ++++- .../tests/test_harness_ai_schema_repair.py | 4 +- sdk/python/tests/test_harness_factory.py | 1 + sdk/python/tests/test_invariants.py | 161 ++++++++---- sdk/python/tests/test_issue_589.py | 2 +- .../tests/test_media_providers_additional.py | 8 +- .../tests/test_memory_bigfiles_coverage.py | 42 ++- sdk/python/tests/test_memory_client_core.py | 4 +- .../tests/test_memory_coverage_additions.py | 12 +- sdk/python/tests/test_memory_events.py | 8 +- .../tests/test_memory_events_additional.py | 67 ++++- sdk/python/tests/test_memory_invariants.py | 22 +- sdk/python/tests/test_memory_performance.py | 82 +++--- .../test_multimodal_response_additional.py | 41 ++- .../tests/test_multimodal_response_cost.py | 18 +- sdk/python/tests/test_pydantic_utils.py | 6 +- .../tests/test_reasoner_path_normalization.py | 4 +- .../test_result_cache_bigfiles_coverage.py | 20 +- sdk/python/tests/test_simulate_trigger.py | 9 +- sdk/python/tests/test_strict_openai_schema.py | 8 +- sdk/python/tests/test_tool_calling.py | 15 +- .../tests/test_tool_calling_error_paths.py | 9 +- sdk/python/tests/test_trigger_context.py | 65 ++--- sdk/python/tests/test_types.py | 14 +- sdk/python/tests/test_vc_generator.py | 9 +- .../tests/test_vc_generator_error_paths.py | 12 +- sdk/python/tests/test_verification.py | 14 +- .../tests/test_workflow_parent_child.py | 75 ++++-- 92 files changed, 1535 insertions(+), 767 deletions(-) diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index ad8f5bf80..57b2e1686 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -659,6 +659,7 @@ def __init__( # one, and a re-import in the same process keeps the same one (since the # same Agent instance is being used). import uuid as _uuid + self.agent_instance_id = _uuid.uuid4().hex # Memory-efficient handler registries (replaces old list-based storage) @@ -734,6 +735,7 @@ def __init__( # before any user-defined reasoners are registered so the path is # always available for the control-plane callback. from .cancel import install_cancel_route + install_cancel_route(self) # Initialize async execution manager (will be lazily created when needed) @@ -920,7 +922,11 @@ def _entry_to_metadata( metadata["accepts_webhook"] = "true" elif accepts_webhook is False: metadata["accepts_webhook"] = "false" - elif isinstance(accepts_webhook, str) and accepts_webhook in ("true", "false", "warn"): + elif isinstance(accepts_webhook, str) and accepts_webhook in ( + "true", + "false", + "warn", + ): metadata["accepts_webhook"] = accepts_webhook else: metadata["accepts_webhook"] = "warn" @@ -1559,8 +1565,7 @@ def _build_agent_metadata(self) -> Optional[Dict[str, Any]]: @property def sessions(self) -> List[Dict[str, Any]]: return [ - entry["definition"].to_dict() - for entry in self._session_registry.values() + entry["definition"].to_dict() for entry in self._session_registry.values() ] def session( @@ -1575,7 +1580,10 @@ def session( tools: Optional[List[str]] = None, tags: Optional[List[str]] = None, metadata: Optional[Dict[str, Any]] = None, - ) -> Callable[[Callable[[RealtimeSession], Awaitable[Any]]], Callable[[RealtimeSession], Awaitable[Any]]]: + ) -> Callable[ + [Callable[[RealtimeSession], Awaitable[Any]]], + Callable[[RealtimeSession], Awaitable[Any]], + ]: """Register a realtime/voice session endpoint. Provider and transport are both explicit; AgentField does not infer or @@ -1596,7 +1604,7 @@ def session( ) def decorator( - func: Callable[[RealtimeSession], Awaitable[Any]] + func: Callable[[RealtimeSession], Awaitable[Any]], ) -> Callable[[RealtimeSession], Awaitable[Any]]: self._session_registry[name] = {"definition": definition, "handler": func} setattr(func, "_agentfield_session", definition) @@ -1995,6 +2003,7 @@ async def run_reasoner() -> Any: # execute_async_with_callback handles its own # cancellation accounting on top of this. from .cancel import register_execution_task + await register_execution_task(self, execution_id_header, task) return JSONResponse( status_code=202, @@ -2014,6 +2023,7 @@ async def run_reasoner() -> Any: register_execution_task, deregister_execution, ) + sync_task = asyncio.create_task(run_reasoner()) await register_execution_task(self, execution_id_header, sync_task) try: @@ -2084,7 +2094,7 @@ async def tracked_func(*args, **kwargs): vc_setting = self._effective_component_vc_setting( reasoner_id, self._reasoner_vc_overrides ) - + self._reasoner_registry[reasoner_id] = ReasonerEntry( id=reasoner_id, func=func, @@ -2118,7 +2128,9 @@ async def tracked_func(*args, **kwargs): return decorator - def _detect_and_unwrap_trigger_envelope(self, payload_dict: Dict[str, Any]) -> tuple[Dict[str, Any], Optional[Any]]: + def _detect_and_unwrap_trigger_envelope( + self, payload_dict: Dict[str, Any] + ) -> tuple[Dict[str, Any], Optional[Any]]: """ Detect dispatcher webhook envelope {event: ..., _meta: ...} and unwrap it. Returns (unwrapped_input, trigger_context_or_none). @@ -2127,23 +2139,25 @@ def _detect_and_unwrap_trigger_envelope(self, payload_dict: Dict[str, Any]) -> t # Check if this looks like a dispatcher envelope if not isinstance(payload_dict, dict): return payload_dict, None - + if "event" in payload_dict and "_meta" in payload_dict: # This is a dispatcher envelope event_data = payload_dict.get("event", {}) meta_data = payload_dict.get("_meta", {}) - + # Parse metadata into TriggerContext try: from datetime import datetime from .triggers import TriggerContext - + received_at_str = meta_data.get("received_at", "") if received_at_str: - received_at = datetime.fromisoformat(received_at_str.replace('Z', '+00:00')) + received_at = datetime.fromisoformat( + received_at_str.replace("Z", "+00:00") + ) else: received_at = datetime.utcnow() - + trigger_ctx = TriggerContext( trigger_id=meta_data.get("trigger_id", ""), source=meta_data.get("source", ""), @@ -2157,15 +2171,17 @@ def _detect_and_unwrap_trigger_envelope(self, payload_dict: Dict[str, Any]) -> t except Exception: # If parsing fails, return raw envelope for compatibility return payload_dict, None - + # Not an envelope return payload_dict, None - def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict) -> dict: + def _apply_trigger_transform( + self, trigger_ctx, bindings: list, input_data: dict + ) -> dict: """ Match trigger context against reasoner bindings and apply transform if found. Returns transformed input or original input if no match. - + Matching logic: 1. Find bindings where binding.source == trigger_ctx.source 2. Check event_type: binding.types empty OR trigger_ctx.event_type matches (exact or prefix) @@ -2173,21 +2189,21 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict 4. Apply transform if binding has one """ from .triggers import EventTrigger - + if not bindings or not trigger_ctx: return input_data - + # Find best-matching binding best_match = None best_specificity = -1 # -1 = no match, 0 = broad (empty types), 1+ = specific - + for binding in bindings: if not isinstance(binding, EventTrigger): continue - + if binding.source != trigger_ctx.source: continue - + # Check event_type match if binding.types: # binding has specific types — check for match @@ -2202,21 +2218,23 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict else: # binding accepts all types specificity = 0 - + # This binding matches; is it better than current best? if specificity > best_specificity: best_match = binding best_specificity = specificity - + # Apply transform if found if best_match and best_match.transform: try: return best_match.transform(input_data) except Exception as e: if self.dev_mode: - log_warn(f"Transform failed for {trigger_ctx.source}/{trigger_ctx.event_type}: {e}; using raw input") + log_warn( + f"Transform failed for {trigger_ctx.source}/{trigger_ctx.event_type}: {e}; using raw input" + ) return input_data - + return input_data async def _execute_reasoner_endpoint( @@ -2235,7 +2253,9 @@ async def _execute_reasoner_endpoint( execution_context = ExecutionContext.from_request(request, self.node_id) payload_dict = input_data # Already a dict from runtime validation # Unwrap dispatcher envelope if present (Phase 5 webhook DX) - payload_dict, trigger_context = self._detect_and_unwrap_trigger_envelope(payload_dict) + payload_dict, trigger_context = self._detect_and_unwrap_trigger_envelope( + payload_dict + ) if trigger_context: execution_context.trigger = trigger_context @@ -2285,9 +2305,7 @@ async def _execute_reasoner_endpoint( trigger_bindings = getattr(func, "_reasoner_triggers", []) if execution_context.trigger and trigger_bindings: payload_dict = self._apply_trigger_transform( - execution_context.trigger, - trigger_bindings, - payload_dict + execution_context.trigger, trigger_bindings, payload_dict ) # When invoked via an inbound trigger, the (possibly transformed) @@ -2363,14 +2381,14 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() task = asyncio.create_task( - self.workflow_handler.notify_call_complete( - execution_context.execution_id, - execution_context.workflow_id, - result, - int((end_time - start_time) * 1000), - execution_context, - input_data=payload_dict, - parent_execution_id=execution_context.parent_execution_id, + self.workflow_handler.notify_call_complete( + execution_context.execution_id, + execution_context.workflow_id, + result, + int((end_time - start_time) * 1000), + execution_context, + input_data=payload_dict, + parent_execution_id=execution_context.parent_execution_id, ) ) self._background_tasks.add(task) @@ -2594,6 +2612,7 @@ async def _watchdog() -> None: # plane records status=failed WITHOUT discarding the rich result # (it stores the result payload regardless of terminal status). from .exceptions import ReasonerFailed + if isinstance(exc, ReasonerFailed) and exc.result is not None: payload["result"] = jsonable_encoder(exc.result) log_error(f"Execution {execution_id} failed asynchronously: {exc}") @@ -2615,6 +2634,7 @@ async def _watchdog() -> None: self._pause_clocks.pop(execution_id, None) # Deregister the cancel hook regardless of outcome. from .cancel import deregister_execution + await deregister_execution(self, execution_id) await self._post_execution_status(callback_url, payload, execution_id) @@ -2664,7 +2684,9 @@ def _build_execution_callback_url(self, execution_id: str) -> Optional[str]: + f"/api/v1/executions/{execution_id}/status" ) - def on_change(self, pattern: Union[str, List[str]]) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]: + def on_change( + self, pattern: Union[str, List[str]] + ) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]: """ Decorator to mark a function as a memory event listener. @@ -3138,7 +3160,7 @@ async def _run_async_skill(*args, **kwargs): previous_ctx = self._current_execution_context self._current_execution_context = child_context input_payload = _build_invocation_payload(args, kwargs) - + task = asyncio.create_task( self.workflow_handler.notify_call_start( child_context.execution_id, @@ -4290,6 +4312,7 @@ async def _push_self_running() -> None: ExecutionFailedError, ExecutionTimeoutError, ) + if isinstance( async_error, ( @@ -4419,7 +4442,7 @@ async def _cleanup_async_resources(self) -> None: try: await asyncio.wait_for( asyncio.gather(*self._background_tasks, return_exceptions=True), - timeout = 5 + timeout=5, ) if self.dev_mode: log_debug("Background tasks are cleaned up") @@ -4500,9 +4523,7 @@ async def _send_note(): if self.dev_mode: from agentfield.logger import log_debug - log_debug( - f"NOTE DEBUG: api_base: {self.client.api_base}" - ) + log_debug(f"NOTE DEBUG: api_base: {self.client.api_base}") log_debug( f"NOTE DEBUG: Full URL: {self.client.api_base}/executions/note" ) diff --git a/sdk/python/agentfield/agent_discovery.py b/sdk/python/agentfield/agent_discovery.py index 25891085a..eccc3fd59 100644 --- a/sdk/python/agentfield/agent_discovery.py +++ b/sdk/python/agentfield/agent_discovery.py @@ -2,6 +2,7 @@ from datetime import datetime, timezone from agentfield.logger import log_debug + class _AgentDiscoveryMixin: def _handle_discovery(self) -> dict: """ @@ -48,11 +49,14 @@ def _build_callback_discovery_payload(self) -> Optional[Dict[str, Any]]: "preferred": self.base_url, "callback_candidates": self.callback_candidates, "container": self._is_running_in_container(), - "submitted_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "submitted_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[ + :-3 + ] + + "Z", } return payload - + def _apply_discovery_response(self, payload: Optional[Dict[str, Any]]) -> None: """Update agent networking state from AgentField discovery response.""" @@ -92,4 +96,4 @@ def _apply_discovery_response(self, payload: Optional[Dict[str, Any]]) -> None: normalized.insert(0, resolved) if normalized: - self.callback_candidates = normalized \ No newline at end of file + self.callback_candidates = normalized diff --git a/sdk/python/agentfield/agent_field_handler.py b/sdk/python/agentfield/agent_field_handler.py index e90e2d0bb..4ef47a9bc 100644 --- a/sdk/python/agentfield/agent_field_handler.py +++ b/sdk/python/agentfield/agent_field_handler.py @@ -146,9 +146,7 @@ async def register_with_agentfield_server(self, port: int): f"(pending tags: {pending_tags})" ) await self._wait_for_approval() - log_success( - f"Node '{self.agent.node_id}' tag approval granted" - ) + log_success(f"Node '{self.agent.node_id}' tag approval granted") else: log_success( f"Registered node '{self.agent.node_id}' with AgentField server" @@ -210,15 +208,11 @@ async def _wait_for_approval(self, timeout: int = 300): status = data.get("lifecycle_status", "") if status and status != "pending_approval": return - log_debug( - f"Node '{self.agent.node_id}' still pending approval..." - ) + log_debug(f"Node '{self.agent.node_id}' still pending approval...") except Exception as e: log_debug(f"Polling for approval status failed: {e}") - log_error( - f"Node '{self.agent.node_id}' approval timed out after {timeout}s" - ) + log_error(f"Node '{self.agent.node_id}' approval timed out after {timeout}s") raise TimeoutError( f"Agent '{self.agent.node_id}' tag approval timed out after {timeout} seconds. " "Please approve the agent's tags in the control plane admin UI." @@ -299,8 +293,8 @@ async def send_enhanced_heartbeat(self) -> bool: heartbeat_data = HeartbeatData( status=self.agent._current_status, timestamp=datetime.now().isoformat(), - version=getattr(self.agent, 'version', '') or '', - instance_id=getattr(self.agent, 'agent_instance_id', '') or '', + version=getattr(self.agent, "version", "") or "", + instance_id=getattr(self.agent, "agent_instance_id", "") or "", ) # Send enhanced heartbeat diff --git a/sdk/python/agentfield/agent_schema.py b/sdk/python/agentfield/agent_schema.py index 426cfb505..5f176652e 100644 --- a/sdk/python/agentfield/agent_schema.py +++ b/sdk/python/agentfield/agent_schema.py @@ -1,5 +1,6 @@ from typing import Any, Dict, Union + class _AgentSchemaMixin: def _types_to_json_schema(self, input_types: Dict[str, tuple]) -> Dict: """Convert Python types dict to JSON schema (on-demand generation).""" @@ -18,7 +19,7 @@ def _types_to_json_schema(self, input_types: Dict[str, tuple]) -> Dict: if required: schema["required"] = required return schema - + def _type_to_json_schema(self, typ: type) -> Dict: """Convert a Python type to JSON schema.""" # Handle None/NoneType @@ -158,4 +159,4 @@ def _validate_handler_input( except (ValueError, TypeError) as e: raise ValueError(f"Invalid value for field '{name}': {e}") - return result \ No newline at end of file + return result diff --git a/sdk/python/agentfield/agent_serverless.py b/sdk/python/agentfield/agent_serverless.py index 50671750f..9b58606d7 100644 --- a/sdk/python/agentfield/agent_serverless.py +++ b/sdk/python/agentfield/agent_serverless.py @@ -3,6 +3,7 @@ from agentfield.execution_context import ExecutionContext from agentfield.logger import log_warn + class _AgentServerlessMixin: def handle_serverless( self, event: dict, adapter: Optional[Callable] = None @@ -163,4 +164,4 @@ def lambda_handler(event, context): return {"statusCode": 500, "body": {"error": str(e)}} finally: # Clean up execution context - self._current_execution_context = None \ No newline at end of file + self._current_execution_context = None diff --git a/sdk/python/agentfield/agent_vc.py b/sdk/python/agentfield/agent_vc.py index c14ff8832..9b89208ab 100644 --- a/sdk/python/agentfield/agent_vc.py +++ b/sdk/python/agentfield/agent_vc.py @@ -1,4 +1,3 @@ - from typing import Any, Dict, Optional from agentfield.logger import log_debug, log_info, log_warn, log_error @@ -26,7 +25,7 @@ def _initialize_did_system(self): log_error(f"Failed to initialize DID system: {e}") self.did_manager = None self.vc_generator = None - + def _populate_execution_context_with_did( self, execution_context, did_execution_context ): @@ -46,7 +45,7 @@ def _populate_execution_context_with_did( def _agent_vc_default(self) -> bool: """Resolve the agent-level VC default, falling back to enabled.""" return True if self._agent_vc_enabled is None else self._agent_vc_enabled - + def _set_reasoner_vc_override( self, reasoner_id: str, value: Optional[bool] ) -> None: @@ -67,7 +66,7 @@ def _effective_component_vc_setting( if component_id in overrides: return overrides[component_id] return self._agent_vc_default() - + def _should_generate_vc( self, component_id: str, overrides: Dict[str, bool] ) -> bool: @@ -78,7 +77,7 @@ def _should_generate_vc( ): return False return self._effective_component_vc_setting(component_id, overrides) - + def _build_agent_metadata(self) -> Optional[Dict[str, Any]]: """Build agent metadata (description, tags, author) for registration payload.""" metadata: Dict[str, Any] = {} @@ -89,7 +88,7 @@ def _build_agent_metadata(self) -> Optional[Dict[str, Any]]: if self.author: metadata["author"] = self.author return metadata if metadata else None - + def _build_vc_metadata(self) -> Dict[str, Any]: """Produce a serializable VC policy snapshot for control-plane visibility.""" effective_reasoners = { @@ -114,7 +113,7 @@ def _build_vc_metadata(self) -> Dict[str, Any]: "effective_reasoners": effective_reasoners, "effective_skills": effective_skills, } - + async def _generate_vc_async( self, vc_generator, @@ -152,4 +151,4 @@ async def _generate_vc_async( if vc: log_info(f"Generated VC {vc.vc_id} for {function_name}") except Exception as e: - log_warn(f"Failed to generate VC for {function_name}: {e}") \ No newline at end of file + log_warn(f"Failed to generate VC for {function_name}: {e}") diff --git a/sdk/python/agentfield/async_execution_manager.py b/sdk/python/agentfield/async_execution_manager.py index 80ee1b5cc..9d1ec9cb9 100644 --- a/sdk/python/agentfield/async_execution_manager.py +++ b/sdk/python/agentfield/async_execution_manager.py @@ -17,7 +17,12 @@ import aiohttp from .async_config import AsyncConfig -from .execution_state import ExecuteError, ExecutionPriority, ExecutionState, ExecutionStatus +from .execution_state import ( + ExecuteError, + ExecutionPriority, + ExecutionState, + ExecutionStatus, +) from .exceptions import ( AgentFieldClientError, ExecutionCancelledError, @@ -241,9 +246,7 @@ def __init__( # Polling coordination self._polling_task: Optional[asyncio.Task] = None - self._polling_semaphore = LazySemaphore( - lambda: self.config.max_active_polls - ) + self._polling_semaphore = LazySemaphore(lambda: self.config.max_active_polls) self._shutdown_event: Optional[asyncio.Event] = None # Metrics and monitoring @@ -389,7 +392,9 @@ async def submit_execution( # Check circuit breaker if self._is_circuit_breaker_open(): - raise AgentFieldClientError("Circuit breaker is open - too many recent failures") + raise AgentFieldClientError( + "Circuit breaker is open - too many recent failures" + ) # Reserve capacity slot; released once terminal await self._capacity_semaphore.acquire() @@ -413,7 +418,10 @@ async def submit_execution( body_bytes = json.dumps(payload, separators=(",", ":")).encode("utf-8") # Add DID authentication headers if configured - if self._did_authenticator is not None and self._did_authenticator.is_configured: + if ( + self._did_authenticator is not None + and self._did_authenticator.is_configured + ): did_headers = self._did_authenticator.sign_headers(body_bytes) request_headers.update(did_headers) @@ -437,8 +445,14 @@ async def submit_execution( error_body = None body_msg = "" if isinstance(error_body, dict): - body_msg = error_body.get("message") or error_body.get("error") or "" - msg = f"{response.status}, {body_msg}" if body_msg else str(response.status) + body_msg = ( + error_body.get("message") or error_body.get("error") or "" + ) + msg = ( + f"{response.status}, {body_msg}" + if body_msg + else str(response.status) + ) raise ExecuteError(response.status, msg, error_body) result = await response.json() @@ -616,9 +630,7 @@ def _active_elapsed() -> float: async with self._execution_lock: execution = self._executions.get(execution_id) if execution is not None: - previous_pause_clock = getattr( - execution, "_pause_clock", None - ) + previous_pause_clock = getattr(execution, "_pause_clock", None) execution._pause_clock = pause_clock attached_pause_clock = True try: @@ -1216,9 +1228,7 @@ async def _poll_single_execution(self, execution: ExecutionState) -> None: kwargs: Dict[str, Any] = {"timeout": self.config.polling_timeout} if self._auth_headers: kwargs["headers"] = dict(self._auth_headers) - response = await self.connection_manager.request( - "GET", url, **kwargs - ) + response = await self.connection_manager.request("GET", url, **kwargs) duration = time.time() - start_time await self._process_poll_response(execution, response, duration) @@ -1302,10 +1312,10 @@ async def _update_execution_from_status( old_status = execution.status - should_apply_terminal_payload = new_status != old_status or ( - new_status == ExecutionStatus.SUCCEEDED and execution.result is None - ) or ( - new_status == ExecutionStatus.FAILED and not execution.error_message + should_apply_terminal_payload = ( + new_status != old_status + or (new_status == ExecutionStatus.SUCCEEDED and execution.result is None) + or (new_status == ExecutionStatus.FAILED and not execution.error_message) ) # Update status and terminal payloads. Replay-hit executions can be diff --git a/sdk/python/agentfield/client.py b/sdk/python/agentfield/client.py index 3d0f4d528..cb8cc979a 100644 --- a/sdk/python/agentfield/client.py +++ b/sdk/python/agentfield/client.py @@ -42,16 +42,23 @@ def _utc_now_iso() -> str: Use this for all outbound timestamps to ensure consistent UTC-aware formatting across the client rather than naive local datetimes. """ - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + return ( + datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[ + :-3 + ] + + "Z" + ) # --------------------------------------------------------------------------- # Typed response models for approval helpers # --------------------------------------------------------------------------- + @dataclass class ApprovalRequestResponse: """Response from requesting approval for an execution.""" + approval_request_id: str approval_request_url: str @@ -59,6 +66,7 @@ class ApprovalRequestResponse: @dataclass class ApprovalStatusResponse: """Response from polling approval status.""" + status: str # pending, approved, rejected, expired response: Optional[Dict[str, Any]] = None request_url: Optional[str] = None @@ -89,6 +97,7 @@ def changes_requested(self) -> bool: if sys.version_info >= (3, 9): from asyncio import to_thread as _to_thread else: + async def _to_thread(func, *args, **kwargs): """Compatibility shim for asyncio.to_thread on Python 3.8.""" loop = asyncio.get_event_loop() @@ -157,7 +166,9 @@ def __init__( self.api_key = api_key # DID authentication for agent-to-agent calls - self._did_authenticator = DIDAuthenticator(did=did, private_key_jwk=private_key_jwk) + self._did_authenticator = DIDAuthenticator( + did=did, private_key_jwk=private_key_jwk + ) # Async execution components self.async_config = async_config or AsyncConfig.from_environment() @@ -175,7 +186,9 @@ def __init__( AgentFieldClient._init_shared_sync_session() def _generate_id(self, prefix: str) -> str: - timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S") + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%d_%H%M%S" + ) random_suffix = f"{random.getrandbits(32):08x}" return f"{prefix}_{timestamp}_{random_suffix}" @@ -198,9 +211,7 @@ def _build_event_stream_headers( event_headers[key] = value return event_headers - def _sanitize_header_values( - self, headers: Dict[str, Any] - ) -> Dict[str, str]: + def _sanitize_header_values(self, headers: Dict[str, Any]) -> Dict[str, str]: """Ensure all header values are concrete strings for requests/httpx.""" sanitized: Dict[str, str] = {} @@ -400,9 +411,7 @@ def _dedupe(values: Optional[List[str]]) -> List[str]: payload = response.json() if fmt == "compact": compact = CompactDiscoveryResponse.from_dict(payload) - return DiscoveryResult( - format=fmt, raw=raw_body, compact=compact, json=None - ) + return DiscoveryResult(format=fmt, raw=raw_body, compact=compact, json=None) json_payload = DiscoveryResponse.from_dict(payload) return DiscoveryResult(format="json", raw=raw_body, json=json_payload) @@ -494,10 +503,12 @@ def _init_shared_sync_session(cls) -> None: ) session.mount("http://", adapter) session.mount("https://", adapter) - session.headers.update({ - "User-Agent": "AgentFieldSDK/1.0", - "Accept": "application/json", - }) + session.headers.update( + { + "User-Agent": "AgentFieldSDK/1.0", + "Accept": "application/json", + } + ) cls._shared_sync_session = session @classmethod @@ -680,7 +691,12 @@ async def register_agent( "environment": "development", "platform": "python", "region": "local", - "tags": {"sdk_version": importlib.import_module("agentfield").__version__, "language": "python"}, + "tags": { + "sdk_version": importlib.import_module( + "agentfield" + ).__version__, + "language": "python", + }, }, "performance": {"latency_ms": 0, "throughput_ps": 0}, "custom": custom_metadata, @@ -800,7 +816,11 @@ async def restart_execution( body_msg = "" if isinstance(error_body, dict): body_msg = error_body.get("message") or error_body.get("error") or "" - msg = f"{response.status_code}, {body_msg}" if body_msg else str(response.status_code) + msg = ( + f"{response.status_code}, {body_msg}" + if body_msg + else str(response.status_code) + ) raise ExecuteError(response.status_code, msg, error_body) return response.json() @@ -899,7 +919,11 @@ def _submit_execution_sync( body_msg = "" if isinstance(error_body, dict): body_msg = error_body.get("message") or error_body.get("error") or "" - msg = f"{response.status_code}, {body_msg}" if body_msg else str(response.status_code) + msg = ( + f"{response.status_code}, {body_msg}" + if body_msg + else str(response.status_code) + ) raise ExecuteError(response.status_code, msg, error_body) body = response.json() return self._parse_submission(body, final_headers, target) @@ -940,7 +964,11 @@ async def _submit_execution_async( body_msg = "" if isinstance(error_body, dict): body_msg = error_body.get("message") or error_body.get("error") or "" - msg = f"{response.status_code}, {body_msg}" if body_msg else str(response.status_code) + msg = ( + f"{response.status_code}, {body_msg}" + if body_msg + else str(response.status_code) + ) raise ExecuteError(response.status_code, msg, error_body) body = response.json() return self._parse_submission(body, final_headers, target) @@ -1088,7 +1116,9 @@ def _format_execution_result( elif metadata.get("started_at"): metadata["timestamp"] = metadata["started_at"] else: - metadata["timestamp"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + metadata["timestamp"] = datetime.datetime.now( + datetime.timezone.utc + ).isoformat() # Cache successful results for reuse if normalized_status in SUCCESS_STATUSES: @@ -1298,7 +1328,12 @@ async def register_agent_with_status( "environment": "development", "platform": "python", "region": "local", - "tags": {"sdk_version": importlib.import_module("agentfield").__version__, "language": "python"}, + "tags": { + "sdk_version": importlib.import_module( + "agentfield" + ).__version__, + "language": "python", + }, }, "performance": {"latency_ms": 0, "throughput_ps": 0}, "custom": custom_metadata, @@ -1494,9 +1529,7 @@ async def poll_execution_status( logger.error( f"Failed to poll execution status for {execution_id[:8]}...: {e}" ) - raise AgentFieldClientError( - f"Failed to poll execution status: {e}" - ) from e + raise AgentFieldClientError(f"Failed to poll execution status: {e}") from e async def batch_check_statuses( self, execution_ids: List[str] @@ -1661,9 +1694,7 @@ async def cancel_async_execution( raise except Exception as e: logger.error(f"Failed to cancel execution {execution_id[:8]}...: {e}") - raise AgentFieldClientError( - f"Failed to cancel execution: {e}" - ) from e + raise AgentFieldClientError(f"Failed to cancel execution: {e}") from e async def list_async_executions( self, status_filter: Optional[str] = None, limit: Optional[int] = None @@ -1705,9 +1736,7 @@ async def list_async_executions( raise except Exception as e: logger.error(f"Failed to list async executions: {e}") - raise AgentFieldClientError( - f"Failed to list async executions: {e}" - ) from e + raise AgentFieldClientError(f"Failed to list async executions: {e}") from e async def get_async_execution_metrics(self) -> Dict[str, Any]: """ @@ -1822,13 +1851,13 @@ async def request_approval( response = await client.post( url, json=body, - headers=self._sanitize_header_values(self._get_headers_with_context(None)), + headers=self._sanitize_header_values( + self._get_headers_with_context(None) + ), timeout=30, ) except Exception as exc: - raise AgentFieldClientError( - f"Failed to request approval: {exc}" - ) from exc + raise AgentFieldClientError(f"Failed to request approval: {exc}") from exc if response.status_code >= 400: raise AgentFieldClientError( @@ -1879,14 +1908,18 @@ async def notify_awaiter_status( body: Dict[str, Any] = {"status": status} if reason: body["reason"] = reason - url = f"{self.api_base}/agents/{node_id}/executions/{execution_id}/awaiter-status" + url = ( + f"{self.api_base}/agents/{node_id}/executions/{execution_id}/awaiter-status" + ) try: client = await self.get_async_http_client() response = await client.post( url, json=body, - headers=self._sanitize_header_values(self._get_headers_with_context(None)), + headers=self._sanitize_header_values( + self._get_headers_with_context(None) + ), timeout=10, ) except Exception as exc: @@ -1922,7 +1955,9 @@ async def get_approval_status( client = await self.get_async_http_client() response = await client.get( url, - headers=self._sanitize_header_values(self._get_headers_with_context(None)), + headers=self._sanitize_header_values( + self._get_headers_with_context(None) + ), timeout=30, ) except Exception as exc: diff --git a/sdk/python/agentfield/decorator_metadata.py b/sdk/python/agentfield/decorator_metadata.py index b77099858..b4e493687 100644 --- a/sdk/python/agentfield/decorator_metadata.py +++ b/sdk/python/agentfield/decorator_metadata.py @@ -31,7 +31,9 @@ def stage_trigger(func: Callable[..., Any], trigger: Any) -> None: existing.append(trigger) -def split_direct_registration_arg(value: Any) -> tuple[Optional[Callable[..., Any]], Any]: +def split_direct_registration_arg( + value: Any, +) -> tuple[Optional[Callable[..., Any]], Any]: """Separate ``@decorator`` usage from ``@decorator(...)`` options.""" if value and (inspect.isfunction(value) or inspect.ismethod(value)): return value, None diff --git a/sdk/python/agentfield/did_auth.py b/sdk/python/agentfield/did_auth.py index 792743020..55e80d11c 100644 --- a/sdk/python/agentfield/did_auth.py +++ b/sdk/python/agentfield/did_auth.py @@ -46,7 +46,11 @@ def _load_ed25519_private_key(private_key_jwk: str): ) try: - jwk = json.loads(private_key_jwk) if isinstance(private_key_jwk, str) else private_key_jwk + jwk = ( + json.loads(private_key_jwk) + if isinstance(private_key_jwk, str) + else private_key_jwk + ) # Verify key type if jwk.get("kty") != "OKP" or jwk.get("crv") != "Ed25519": @@ -159,7 +163,9 @@ class DIDAuthenticator: for creating authenticated request headers. """ - def __init__(self, did: Optional[str] = None, private_key_jwk: Optional[str] = None): + def __init__( + self, did: Optional[str] = None, private_key_jwk: Optional[str] = None + ): """ Initialize DID authenticator. diff --git a/sdk/python/agentfield/did_manager.py b/sdk/python/agentfield/did_manager.py index ff148e71b..9b5e1862f 100644 --- a/sdk/python/agentfield/did_manager.py +++ b/sdk/python/agentfield/did_manager.py @@ -68,7 +68,10 @@ class DIDManager: """ def __init__( - self, agentfield_server_url: str, agent_node_id: str, api_key: Optional[str] = None + self, + agentfield_server_url: str, + agent_node_id: str, + api_key: Optional[str] = None, ): """ Initialize DID Manager. diff --git a/sdk/python/agentfield/exceptions.py b/sdk/python/agentfield/exceptions.py index 94af1cc7f..df1bd074b 100644 --- a/sdk/python/agentfield/exceptions.py +++ b/sdk/python/agentfield/exceptions.py @@ -5,9 +5,11 @@ class AgentFieldError(Exception): """Base exception for all AgentField SDK errors.""" - def __init__(self, message:str): + + def __init__(self, message: str): super().__init__(message) + class AgentFieldClientError(AgentFieldError): """Error communicating with the AgentField control plane.""" diff --git a/sdk/python/agentfield/execution_state.py b/sdk/python/agentfield/execution_state.py index 18a9cba7d..2d48a2552 100644 --- a/sdk/python/agentfield/execution_state.py +++ b/sdk/python/agentfield/execution_state.py @@ -272,7 +272,15 @@ def update_status( # Update metrics based on status change current_time = time.time() - if old_status in {ExecutionStatus.PENDING, ExecutionStatus.QUEUED, ExecutionStatus.WAITING} and status == ExecutionStatus.RUNNING: + if ( + old_status + in { + ExecutionStatus.PENDING, + ExecutionStatus.QUEUED, + ExecutionStatus.WAITING, + } + and status == ExecutionStatus.RUNNING + ): self.metrics.start_time = current_time elif status in { ExecutionStatus.SUCCEEDED, diff --git a/sdk/python/agentfield/harness/_cli.py b/sdk/python/agentfield/harness/_cli.py index 63e4be44e..606b61bb2 100644 --- a/sdk/python/agentfield/harness/_cli.py +++ b/sdk/python/agentfield/harness/_cli.py @@ -150,9 +150,7 @@ def _kill_group() -> None: await proc.wait() if idle_timed_out: - raise TimeoutError( - f"CLI command made no progress for {idle}s: {' '.join(cmd)}" - ) + raise TimeoutError(f"CLI command made no progress for {idle}s: {' '.join(cmd)}") if timed_out: raise TimeoutError(f"CLI command timed out after {timeout}s: {' '.join(cmd)}") diff --git a/sdk/python/agentfield/http_connection_manager.py b/sdk/python/agentfield/http_connection_manager.py index 3a34ed2fc..657cdcad4 100644 --- a/sdk/python/agentfield/http_connection_manager.py +++ b/sdk/python/agentfield/http_connection_manager.py @@ -228,7 +228,9 @@ async def get_session(self): AgentFieldClientError: If manager is not started or is closed """ if self._session is None: - raise AgentFieldClientError("ConnectionManager is not started. Call start() first.") + raise AgentFieldClientError( + "ConnectionManager is not started. Call start() first." + ) if self._closed: raise AgentFieldClientError("ConnectionManager is closed") diff --git a/sdk/python/agentfield/media_providers.py b/sdk/python/agentfield/media_providers.py index 2bbec0607..0ce9dfa5d 100644 --- a/sdk/python/agentfield/media_providers.py +++ b/sdk/python/agentfield/media_providers.py @@ -430,9 +430,7 @@ async def generate_audio( # Merge additional kwargs fal_args.update(kwargs) - response_format = kwargs.get("output_format") or kwargs.get( - "response_format" - ) + response_format = kwargs.get("output_format") or kwargs.get("response_format") output_format = response_format if isinstance(response_format, str) else format try: @@ -942,8 +940,7 @@ async def generate_image( user_content: Any = prompt if image_urls: user_content = [{"type": "text", "text": prompt}] + [ - {"type": "image_url", "image_url": {"url": url}} - for url in image_urls + {"type": "image_url", "image_url": {"url": url}} for url in image_urls ] body: Dict[str, Any] = { @@ -992,9 +989,7 @@ def add_image_url(url: Optional[str]) -> None: b64_json = None if url.startswith("data:image/") and "base64," in url: b64_json = url.split("base64,", 1)[1] - images.append( - ImageOutput(url=url, b64_json=b64_json, revised_prompt=None) - ) + images.append(ImageOutput(url=url, b64_json=b64_json, revised_prompt=None)) for choice in payload.get("choices", []) or []: message = choice.get("message", {}) or {} @@ -1331,9 +1326,7 @@ async def _stream_openrouter_audio( buf += raw_chunk while b"\n" in buf: raw_line, buf = buf.split(b"\n", 1) - decoded = raw_line.decode( - "utf-8", errors="replace" - ).strip() + decoded = raw_line.decode("utf-8", errors="replace").strip() if not decoded.startswith("data: "): continue data_str = decoded[len("data: ") :] @@ -1360,9 +1353,7 @@ async def _stream_openrouter_audio( ) b64_chunks.append(chunk) if audio_delta.get("transcript"): - transcript_parts.append( - audio_delta["transcript"] - ) + transcript_parts.append(audio_delta["transcript"]) b64_full = "".join(b64_chunks) transcript = "".join(transcript_parts) diff --git a/sdk/python/agentfield/memory.py b/sdk/python/agentfield/memory.py index 46e80683e..afdd1fd73 100644 --- a/sdk/python/agentfield/memory.py +++ b/sdk/python/agentfield/memory.py @@ -113,6 +113,7 @@ if sys.version_info >= (3, 9): from asyncio import to_thread as _to_thread else: + async def _to_thread(func, *args, **kwargs): """Compatibility shim for asyncio.to_thread on Python 3.8.""" loop = asyncio.get_event_loop() @@ -184,7 +185,11 @@ async def _async_request(self, method: str, url: str, **kwargs): return await _to_thread(requests.request, method, url, **kwargs) async def set( - self, key: str, data: Any, scope: Optional[str] = None, scope_id: Optional[str] = None + self, + key: str, + data: Any, + scope: Optional[str] = None, + scope_id: Optional[str] = None, ) -> None: """ Set a memory value with automatic scoping. @@ -423,13 +428,9 @@ async def delete_vector( except MemoryAccessError: raise except Exception as e: - raise MemoryAccessError( - f"Failed to delete vector key '{key}': {e}" - ) from e + raise MemoryAccessError(f"Failed to delete vector key '{key}': {e}") from e - async def list_keys( - self, scope: str, scope_id: Optional[str] = None - ) -> List[str]: + async def list_keys(self, scope: str, scope_id: Optional[str] = None) -> List[str]: """ List all keys in a specific scope. @@ -550,9 +551,7 @@ async def exists(self, key: str) -> bool: async def delete(self, key: str) -> None: """Delete a value from this specific scope.""" - await self.memory_client.delete( - key, scope=self.scope, scope_id=self.scope_id - ) + await self.memory_client.delete(key, scope=self.scope, scope_id=self.scope_id) async def list_keys(self) -> List[str]: """List all keys in this specific scope.""" diff --git a/sdk/python/agentfield/memory_events.py b/sdk/python/agentfield/memory_events.py index 5771e5bf3..10938d142 100644 --- a/sdk/python/agentfield/memory_events.py +++ b/sdk/python/agentfield/memory_events.py @@ -85,9 +85,9 @@ class MemoryEventClient: def __init__(self, base_url: str, execution_context, api_key: Optional[str] = None): if base_url.startswith("https://"): - self.base_url = "wss://" + base_url[len("https://"):] + self.base_url = "wss://" + base_url[len("https://") :] elif base_url.startswith("http://"): - self.base_url = "ws://" + base_url[len("http://"):] + self.base_url = "ws://" + base_url[len("http://") :] else: self.base_url = base_url self.execution_context = execution_context @@ -347,9 +347,9 @@ async def history( # Make request to history endpoint if self.base_url.startswith("wss://"): - http_url = "https://" + self.base_url[len("wss://"):] + http_url = "https://" + self.base_url[len("wss://") :] elif self.base_url.startswith("ws://"): - http_url = "http://" + self.base_url[len("ws://"):] + http_url = "http://" + self.base_url[len("ws://") :] else: http_url = self.base_url response = await client.get( @@ -397,9 +397,9 @@ async def history( # Make request to history endpoint if self.base_url.startswith("wss://"): - http_url = "https://" + self.base_url[len("wss://"):] + http_url = "https://" + self.base_url[len("wss://") :] elif self.base_url.startswith("ws://"): - http_url = "http://" + self.base_url[len("ws://"):] + http_url = "http://" + self.base_url[len("ws://") :] else: http_url = self.base_url response = requests.get( diff --git a/sdk/python/agentfield/router.py b/sdk/python/agentfield/router.py index f6cebb85b..78af022a6 100644 --- a/sdk/python/agentfield/router.py +++ b/sdk/python/agentfield/router.py @@ -136,7 +136,7 @@ def __getattr__(self, name: str) -> Any: """ # Avoid infinite recursion by accessing _agent through object.__getattribute__ try: - agent = object.__getattribute__(self, '_agent') + agent = object.__getattribute__(self, "_agent") except AttributeError: raise AgentFieldClientError( "Router not attached to an agent. Call Agent.include_router(router) first." diff --git a/sdk/python/agentfield/session_transport.py b/sdk/python/agentfield/session_transport.py index 06307f4be..5b69adf91 100644 --- a/sdk/python/agentfield/session_transport.py +++ b/sdk/python/agentfield/session_transport.py @@ -20,7 +20,9 @@ class SessionTransportError(ValueError): """Raised when a session provider/transport pair is unsupported.""" - def __init__(self, provider: str, transport: str, supported: FrozenSet[str]) -> None: + def __init__( + self, provider: str, transport: str, supported: FrozenSet[str] + ) -> None: self.provider = provider self.transport = transport self.supported = supported @@ -42,7 +44,9 @@ def normalize_session_transport_value(value: str) -> str: return value.strip().lower().replace("-", "_") -def validate_session_transport(provider: str, transport: str) -> SessionTransportCapability: +def validate_session_transport( + provider: str, transport: str +) -> SessionTransportCapability: """Validate an explicit session provider/transport pair. This is intended to run in SDK registration paths and again in the control @@ -53,9 +57,13 @@ def validate_session_transport(provider: str, transport: str) -> SessionTranspor normalized_transport = normalize_session_transport_value(transport) if not normalized_provider: - raise ValueError("Session provider is required; AgentField does not infer providers.") + raise ValueError( + "Session provider is required; AgentField does not infer providers." + ) if not normalized_transport: - raise ValueError("Session transport is required; AgentField does not infer transports.") + raise ValueError( + "Session transport is required; AgentField does not infer transports." + ) supported = SUPPORTED_SESSION_TRANSPORTS.get(normalized_provider) if supported is None: @@ -66,7 +74,9 @@ def validate_session_transport(provider: str, transport: str) -> SessionTranspor ) if normalized_transport not in supported: - raise SessionTransportError(normalized_provider, normalized_transport, supported) + raise SessionTransportError( + normalized_provider, normalized_transport, supported + ) return SessionTransportCapability( provider=normalized_provider, diff --git a/sdk/python/agentfield/tool_calling.py b/sdk/python/agentfield/tool_calling.py index 6afd439c7..5a2715b0a 100644 --- a/sdk/python/agentfield/tool_calling.py +++ b/sdk/python/agentfield/tool_calling.py @@ -517,7 +517,9 @@ async def execute_tool_call_loop( # Convert invocation_target format to agent.call() format call_target = _invocation_target_to_call_target(invocation_target) - log_debug(f"Tool call [{total_calls}]: {func_name} -> {call_target}({json.dumps(func_args)})") + log_debug( + f"Tool call [{total_calls}]: {func_name} -> {call_target}({json.dumps(func_args)})" + ) start_time = time.monotonic() try: diff --git a/sdk/python/agentfield/verification.py b/sdk/python/agentfield/verification.py index f0d171d7f..ea653457d 100644 --- a/sdk/python/agentfield/verification.py +++ b/sdk/python/agentfield/verification.py @@ -112,7 +112,9 @@ async def refresh(self) -> bool: self.revoked_dids = set(data.get("revoked_dids", [])) logger.debug(f"Refreshed {len(self.revoked_dids)} revoked DIDs") else: - logger.warning(f"Failed to fetch revocations: HTTP {resp.status}") + logger.warning( + f"Failed to fetch revocations: HTTP {resp.status}" + ) success = False except Exception as e: logger.warning(f"Failed to fetch revocations: {e}") @@ -128,9 +130,13 @@ async def refresh(self) -> bool: if resp.status == 200: data = await resp.json() self.registered_dids = set(data.get("registered_dids", [])) - logger.debug(f"Refreshed {len(self.registered_dids)} registered DIDs") + logger.debug( + f"Refreshed {len(self.registered_dids)} registered DIDs" + ) else: - logger.warning(f"Failed to fetch registered DIDs: HTTP {resp.status}") + logger.warning( + f"Failed to fetch registered DIDs: HTTP {resp.status}" + ) success = False except Exception as e: logger.warning(f"Failed to fetch registered DIDs: {e}") @@ -147,9 +153,13 @@ async def refresh(self) -> bool: data = await resp.json() self.admin_public_key_jwk = data.get("public_key_jwk") self.issuer_did = data.get("issuer_did") - logger.debug(f"Refreshed admin public key (issuer: {self.issuer_did})") + logger.debug( + f"Refreshed admin public key (issuer: {self.issuer_did})" + ) else: - logger.warning(f"Failed to fetch admin public key: HTTP {resp.status}") + logger.warning( + f"Failed to fetch admin public key: HTTP {resp.status}" + ) success = False except Exception as e: logger.warning(f"Failed to fetch admin public key: {e}") @@ -222,16 +232,22 @@ def verify_signature( ts = int(timestamp) now = int(time.time()) if abs(now - ts) > self.timestamp_window: - logger.debug(f"Timestamp expired: {now - ts}s drift (window: {self.timestamp_window}s)") + logger.debug( + f"Timestamp expired: {now - ts}s drift (window: {self.timestamp_window}s)" + ) return False except (ValueError, TypeError): logger.debug("Invalid timestamp format") return False try: - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PublicKey, + ) except ImportError: - logger.warning("cryptography library not available for signature verification") + logger.warning( + "cryptography library not available for signature verification" + ) return False try: @@ -272,12 +288,14 @@ def _resolve_public_key(self, caller_did: str) -> Optional[bytes]: """ if caller_did.startswith("did:key:z"): try: - encoded = caller_did[len("did:key:z"):] + encoded = caller_did[len("did:key:z") :] decoded = base64.urlsafe_b64decode(encoded + "==") # Verify Ed25519 multicodec prefix: 0xed, 0x01 if len(decoded) >= 34 and decoded[0] == 0xED and decoded[1] == 0x01: return decoded[2:34] - logger.debug(f"Invalid multicodec prefix in did:key: {decoded[:2].hex()}") + logger.debug( + f"Invalid multicodec prefix in did:key: {decoded[:2].hex()}" + ) return None except Exception as e: logger.debug(f"Failed to decode did:key public key: {e}") @@ -338,12 +356,16 @@ def evaluate_policy( # Check if caller tags match policy_caller_tags = policy.get("caller_tags", []) - if policy_caller_tags and not any(t in caller_tags for t in policy_caller_tags): + if policy_caller_tags and not any( + t in caller_tags for t in policy_caller_tags + ): continue # Check if target tags match policy_target_tags = policy.get("target_tags", []) - if policy_target_tags and not any(t in target_tags for t in policy_target_tags): + if policy_target_tags and not any( + t in target_tags for t in policy_target_tags + ): continue # Check function allow/deny lists @@ -355,7 +377,9 @@ def evaluate_policy( return False # Check allow list - if allow_functions and not _function_matches(function_name, allow_functions): + if allow_functions and not _function_matches( + function_name, allow_functions + ): continue # Check constraints diff --git a/sdk/python/tests/functional/test_agent_execution.py b/sdk/python/tests/functional/test_agent_execution.py index ef74425a5..888ec2ba9 100644 --- a/sdk/python/tests/functional/test_agent_execution.py +++ b/sdk/python/tests/functional/test_agent_execution.py @@ -56,22 +56,22 @@ def run_agent(): json={"input": {"a": 5, "b": 3}}, ) - assert ( - response.status_code == 200 - ), f"Execution failed: {response.status_code} - {response.text}" + assert response.status_code == 200, ( + f"Execution failed: {response.status_code} - {response.text}" + ) result = response.json() # Verify result - assert ( - "result" in result or "sum" in result - ), f"Missing result in response: {result}" + assert "result" in result or "sum" in result, ( + f"Missing result in response: {result}" + ) # Extract the sum value if "result" in result: if isinstance(result["result"], dict): - assert ( - result["result"]["sum"] == 8 - ), f"Expected sum=8, got {result['result']}" + assert result["result"]["sum"] == 8, ( + f"Expected sum=8, got {result['result']}" + ) else: # Result might be directly in the response pass @@ -289,9 +289,9 @@ def run_agent(): ) # Should return an error status - assert ( - response.status_code >= 400 - ), f"Expected error status, got {response.status_code}" + assert response.status_code >= 400, ( + f"Expected error status, got {response.status_code}" + ) print(f"✅ Error handling test passed: {response.status_code}") diff --git a/sdk/python/tests/functional/test_agent_registration.py b/sdk/python/tests/functional/test_agent_registration.py index d88c37de5..dd864f4e0 100644 --- a/sdk/python/tests/functional/test_agent_registration.py +++ b/sdk/python/tests/functional/test_agent_registration.py @@ -113,15 +113,15 @@ async def hello(ping: str = "ok"): json={"input": {"ping": "ok"}}, ) - assert ( - response.status_code == 200 - ), f"Expected 200, got {response.status_code}: {response.text}" + assert response.status_code == 200, ( + f"Expected 200, got {response.status_code}: {response.text}" + ) result = response.json() # Verify response structure - assert ( - "result" in result or "message" in result - ), f"Unexpected response: {result}" + assert "result" in result or "message" in result, ( + f"Unexpected response: {result}" + ) print(f"✅ Agent registered successfully: {result}") @@ -146,9 +146,9 @@ async def ping(): ) response = agent_client.get("/health") - assert ( - response.status_code == 200 - ), f"Health check failed: {response.status_code}" + assert response.status_code == 200, ( + f"Health check failed: {response.status_code}" + ) health_data = response.json() # Verify health response structure @@ -186,9 +186,9 @@ async def identify(ping: str = "ok", agent_index=i): json={"input": {"ping": "ok"}}, ) - assert ( - response.status_code == 200 - ), f"Agent {i} not registered: {response.status_code}" + assert response.status_code == 200, ( + f"Agent {i} not registered: {response.status_code}" + ) print(f"✅ Agent {i} registered and responding") finally: diff --git a/sdk/python/tests/functional/test_agent_to_agent_call.py b/sdk/python/tests/functional/test_agent_to_agent_call.py index ab4d44b65..5ba127b4f 100644 --- a/sdk/python/tests/functional/test_agent_to_agent_call.py +++ b/sdk/python/tests/functional/test_agent_to_agent_call.py @@ -87,9 +87,9 @@ def run_agent_b(): json={"input": {"value": 21}}, ) - assert ( - response.status_code == 200 - ), f"Call failed: {response.status_code} - {response.text}" + assert response.status_code == 200, ( + f"Call failed: {response.status_code} - {response.text}" + ) result = response.json() print(f"✅ Agent-to-agent call successful: {result}") @@ -97,9 +97,9 @@ def run_agent_b(): # Verify the result shows delegation happened # Expected: 21 * 2 = 42 result_str = str(result) - assert ( - "42" in result_str or "delegated" in result_str - ), f"Unexpected result: {result}" + assert "42" in result_str or "delegated" in result_str, ( + f"Unexpected result: {result}" + ) finally: pass @@ -194,18 +194,18 @@ def runner(): json={"input": {"value": 10}}, ) - assert ( - response.status_code == 200 - ), f"Chain call failed: {response.status_code} - {response.text}" + assert response.status_code == 200, ( + f"Chain call failed: {response.status_code} - {response.text}" + ) result = response.json() print(f"✅ Chain of agent calls successful: {result}") # Expected calculation: (10 + 5) * 2 = 30 result_str = str(result) - assert ( - "30" in result_str or "final" in result_str - ), f"Unexpected result: {result}" + assert "30" in result_str or "final" in result_str, ( + f"Unexpected result: {result}" + ) finally: pass diff --git a/sdk/python/tests/functional/test_ai_integration.py b/sdk/python/tests/functional/test_ai_integration.py index ee736db50..c4b8f8e45 100644 --- a/sdk/python/tests/functional/test_ai_integration.py +++ b/sdk/python/tests/functional/test_ai_integration.py @@ -67,18 +67,18 @@ def run_agent(): json={"input": {"question": "What is 2+2?"}}, ) - assert ( - response.status_code == 200 - ), f"AI call failed: {response.status_code} - {response.text}" + assert response.status_code == 200, ( + f"AI call failed: {response.status_code} - {response.text}" + ) result = response.json() print(f"✅ LLM call successful: {result}") # Verify we got a response with an answer result_str = str(result).lower() - assert ( - "answer" in result_str or "4" in result_str - ), f"Expected answer in response: {result}" + assert "answer" in result_str or "4" in result_str, ( + f"Expected answer in response: {result}" + ) finally: pass diff --git a/sdk/python/tests/helpers.py b/sdk/python/tests/helpers.py index 71feaf035..e35dd8dd0 100644 --- a/sdk/python/tests/helpers.py +++ b/sdk/python/tests/helpers.py @@ -262,7 +262,9 @@ def create_test_agent( memory_store: Dict[str, Any] = {} class _FakeAgentFieldClient(DummyAgentFieldClient): - def __init__(self, base_url: str, async_config: Any = None, api_key: Optional[str] = None): + def __init__( + self, base_url: str, async_config: Any = None, api_key: Optional[str] = None + ): super().__init__() self.base_url = base_url self.api_base = f"{base_url}/api/v1" @@ -382,7 +384,9 @@ def decorator(func): return decorator class _FakeDIDManager: - def __init__(self, agentfield_server: str, node: str, api_key: Optional[str] = None): + def __init__( + self, agentfield_server: str, node: str, api_key: Optional[str] = None + ): self.agentfield_server = agentfield_server self.node_id = node self.api_key = api_key diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 91e4bb7df..1b59c8670 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -210,9 +210,9 @@ class AgentRuntime: @pytest.fixture -def run_agent() -> ( - Generator[Callable[[Agent, Optional[int]], AgentRuntime], None, None] -): +def run_agent() -> Generator[ + Callable[[Agent, Optional[int]], AgentRuntime], None, None +]: runtimes: list[AgentRuntime] = [] def _start(agent: Agent, port: Optional[int] = None) -> AgentRuntime: diff --git a/sdk/python/tests/mypy_reveal_simulate_schedule.py b/sdk/python/tests/mypy_reveal_simulate_schedule.py index ee26bf289..2e2d8fb9c 100644 --- a/sdk/python/tests/mypy_reveal_simulate_schedule.py +++ b/sdk/python/tests/mypy_reveal_simulate_schedule.py @@ -18,6 +18,7 @@ def handler(input: object) -> str: async def async_handler(input: object) -> str: return "ok" + handler._reasoner_triggers = [] # type: ignore[attr-defined] async_handler._reasoner_triggers = [] # type: ignore[attr-defined] diff --git a/sdk/python/tests/mypy_reveal_simulate_trigger.py b/sdk/python/tests/mypy_reveal_simulate_trigger.py index a390c2d23..a9f7a3f87 100644 --- a/sdk/python/tests/mypy_reveal_simulate_trigger.py +++ b/sdk/python/tests/mypy_reveal_simulate_trigger.py @@ -18,6 +18,7 @@ def handler(input: object) -> str: async def async_handler(input: object) -> str: return "hello" + handler._reasoner_triggers = [] # type: ignore[attr-defined] async_handler._reasoner_triggers = [] # type: ignore[attr-defined] diff --git a/sdk/python/tests/test_accepts_webhook.py b/sdk/python/tests/test_accepts_webhook.py index 766c6ffc0..15bbb1976 100644 --- a/sdk/python/tests/test_accepts_webhook.py +++ b/sdk/python/tests/test_accepts_webhook.py @@ -9,7 +9,7 @@ @pytest.mark.unit async def test_accepts_webhook_default_is_warn(): """Without triggers or explicit accepts_webhook, default should be 'warn'.""" - + @reasoner async def no_triggers(x: str) -> dict: return {"result": x} @@ -146,9 +146,7 @@ def test_agent_reasoner_preserves_inner_reasoner_trigger_metadata(): async def stacked_no_webhook_trigger(x: str) -> dict: return {"x": x} - metadata = next( - r for r in app.reasoners if r["id"] == "stacked_no_webhook_trigger" - ) + metadata = next(r for r in app.reasoners if r["id"] == "stacked_no_webhook_trigger") assert metadata["accepts_webhook"] == "false" assert metadata["triggers"][0]["source"] == "github" diff --git a/sdk/python/tests/test_agent_ai_coverage_additions.py b/sdk/python/tests/test_agent_ai_coverage_additions.py index 91208f57a..345181885 100644 --- a/sdk/python/tests/test_agent_ai_coverage_additions.py +++ b/sdk/python/tests/test_agent_ai_coverage_additions.py @@ -20,7 +20,9 @@ def agent_with_ai(): @pytest.mark.asyncio -async def test_generate_tts_audio_uses_iterable_binary_response(monkeypatch, agent_with_ai): +async def test_generate_tts_audio_uses_iterable_binary_response( + monkeypatch, agent_with_ai +): class LiteLLMStub: async def aspeech(self, **kwargs): return iter([b"ab", b"cd"]) @@ -35,7 +37,9 @@ async def aspeech(self, **kwargs): @pytest.mark.asyncio -async def test_generate_openai_direct_audio_returns_text_only_when_api_key_missing(agent_with_ai): +async def test_generate_openai_direct_audio_returns_text_only_when_api_key_missing( + agent_with_ai, +): agent_with_ai.ai_config.get_litellm_params = lambda **kwargs: {} result = await AgentAI(agent_with_ai)._generate_openai_direct_audio("hello") @@ -45,7 +49,9 @@ async def test_generate_openai_direct_audio_returns_text_only_when_api_key_missi @pytest.mark.asyncio -async def test_ai_generate_image_uses_configured_default_model(monkeypatch, agent_with_ai): +async def test_ai_generate_image_uses_configured_default_model( + monkeypatch, agent_with_ai +): captured = {} ai = AgentAI(agent_with_ai) diff --git a/sdk/python/tests/test_agent_ai_deadlock_recovery.py b/sdk/python/tests/test_agent_ai_deadlock_recovery.py index 9512dd11b..7b0d2a19d 100644 --- a/sdk/python/tests/test_agent_ai_deadlock_recovery.py +++ b/sdk/python/tests/test_agent_ai_deadlock_recovery.py @@ -59,7 +59,9 @@ def __init__(self): self.response_format = "auto" self.fallback_models = [] self.final_fallback_model = None - self.enable_rate_limit_retry = False # bypass retries; we want to see raw behavior + self.enable_rate_limit_retry = ( + False # bypass retries; we want to see raw behavior + ) self.rate_limit_max_retries = 0 self.rate_limit_base_delay = 0.0 self.rate_limit_max_delay = 0.0 @@ -205,11 +207,11 @@ async def test_hanging_acompletion_triggers_timeout_and_pool_reset( ): """The smoking gun. Reproduces the production deadlock in miniature: - 1. acompletion hangs (asyncio.Event that never sets, like a half-closed - httpx socket). - 2. The asyncio.wait_for safety net at 2 × llm_call_timeout fires. - 3. _reset_litellm_http_clients is invoked (cached clients become None). - 4. A subsequent acompletion call returns successfully (the fix worked). + 1. acompletion hangs (asyncio.Event that never sets, like a half-closed + httpx socket). + 2. The asyncio.wait_for safety net at 2 × llm_call_timeout fires. + 3. _reset_litellm_http_clients is invoked (cached clients become None). + 4. A subsequent acompletion call returns successfully (the fix worked). """ call_count = {"n": 0} never_set = asyncio.Event() # never set → simulates a hung HTTP read @@ -403,7 +405,9 @@ async def test_parallel_hangs_some_succeed_some_recover_via_pool_reset( return_exceptions=True, ) successes = [r for r in results if not isinstance(r, BaseException)] - timeouts = [r for r in results if isinstance(r, (TimeoutError, asyncio.TimeoutError))] + timeouts = [ + r for r in results if isinstance(r, (TimeoutError, asyncio.TimeoutError)) + ] other_errors = [ r for r in results @@ -433,6 +437,7 @@ async def test_parallel_hangs_some_succeed_some_recover_via_pool_reset( counters["hung_release_event"].set() # Reset the predicate so no further calls hang. counters["started"] = 0 # restart numbering for the second batch + # Now reinstall a side effect that always succeeds. We do this by # swapping acompletion outright — simpler than threading state. async def always_ok(**params): @@ -668,8 +673,16 @@ async def acompletion_side_effect(**params): # Stub out the tool-call loop machinery so we can drive _tool_loop_completion # directly without needing real tool schemas. - async def fake_loop(*, agent, messages, tools, config, needs_lazy_hydration, - litellm_params, make_completion): + async def fake_loop( + *, + agent, + messages, + tools, + config, + needs_lazy_hydration, + litellm_params, + make_completion, + ): params = {**litellm_params, "messages": messages} resp = await make_completion(params) return resp, SimpleNamespace(total_turns=1) @@ -679,7 +692,11 @@ async def fake_loop(*, agent, messages, tools, config, needs_lazy_hydration, ) monkeypatch.setattr( "agentfield.tool_calling._build_tool_config", - lambda tools, agent: ([], SimpleNamespace(max_turns=5, max_tool_calls=10), False), + lambda tools, agent: ( + [], + SimpleNamespace(max_turns=5, max_tool_calls=10), + False, + ), raising=False, ) @@ -799,9 +816,7 @@ class _ProviderError(Exception): async def acompletion_side_effect(**params): call_count["n"] += 1 if call_count["n"] < 2: - raise _ProviderError( - "OpenrouterException - Unable to get json response" - ) + raise _ProviderError("OpenrouterException - Unable to get json response") return _make_chat_response("recovered-after-glitch") _install_litellm_stub(monkeypatch, acompletion_side_effect) diff --git a/sdk/python/tests/test_agent_bigfiles_final90.py b/sdk/python/tests/test_agent_bigfiles_final90.py index b507f13cf..d1b33ee78 100644 --- a/sdk/python/tests/test_agent_bigfiles_final90.py +++ b/sdk/python/tests/test_agent_bigfiles_final90.py @@ -94,28 +94,39 @@ def fake_get(url, **kwargs): def test_detect_local_ip_and_container_helpers(monkeypatch): monkeypatch.setattr( - "agentfield.agent.socket.socket", lambda *args, **kwargs: _SocketStub("10.0.0.5") + "agentfield.agent.socket.socket", + lambda *args, **kwargs: _SocketStub("10.0.0.5"), ) assert _detect_local_ip() == "10.0.0.5" monkeypatch.setattr("agentfield.agent.os.path.exists", lambda path: False) monkeypatch.setattr( - "builtins.open", lambda *args, **kwargs: io.StringIO("12:memory:/kubepods"), raising=True + "builtins.open", + lambda *args, **kwargs: io.StringIO("12:memory:/kubepods"), + raising=True, ) assert _is_running_in_container() is True def test_callback_candidate_helpers(monkeypatch): monkeypatch.setattr("agentfield.agent._is_running_in_container", lambda: True) - monkeypatch.setattr("agentfield.agent._detect_container_ip", lambda: "198.51.100.20") + monkeypatch.setattr( + "agentfield.agent._detect_container_ip", lambda: "198.51.100.20" + ) monkeypatch.setattr("agentfield.agent._detect_local_ip", lambda: "10.0.0.8") monkeypatch.setattr("agentfield.agent.socket.gethostname", lambda: "hostbox") monkeypatch.setenv("AGENT_CALLBACK_URL", "callback.internal") monkeypatch.setenv("RAILWAY_SERVICE_NAME", "svc") monkeypatch.setenv("RAILWAY_ENVIRONMENT", "prod") - assert _normalize_candidate("http://[2001:db8::1]", 9000) == "http://[2001:db8::1]:9000" - assert _normalize_candidate("https://example.com:8443", 9000) == "https://example.com:8443" + assert ( + _normalize_candidate("http://[2001:db8::1]", 9000) + == "http://[2001:db8::1]:9000" + ) + assert ( + _normalize_candidate("https://example.com:8443", 9000) + == "https://example.com:8443" + ) candidates = _build_callback_candidates("api.example.com", 8001) assert candidates[0] == "http://api.example.com:8001" diff --git a/sdk/python/tests/test_agent_coverage_additions.py b/sdk/python/tests/test_agent_coverage_additions.py index d6767b2a6..0e4ddf13f 100644 --- a/sdk/python/tests/test_agent_coverage_additions.py +++ b/sdk/python/tests/test_agent_coverage_additions.py @@ -19,7 +19,13 @@ def make_agent(): agent._reasoner_registry = {"summarize": SimpleNamespace(id="summarize")} agent._skill_registry = {"search": SimpleNamespace(id="search")} agent._entry_to_metadata = lambda entry, kind: ( - {"id": entry.id, "input_schema": {"type": "object"}, "output_schema": {}, "memory_config": {}, "tags": ["nlp"]} + { + "id": entry.id, + "input_schema": {"type": "object"}, + "output_schema": {}, + "memory_config": {}, + "tags": ["nlp"], + } if kind == "reasoner" else {"id": entry.id, "input_schema": {}, "tags": ["tools"]} ) diff --git a/sdk/python/tests/test_agent_integration.py b/sdk/python/tests/test_agent_integration.py index 20e59e4f6..254305c0b 100644 --- a/sdk/python/tests/test_agent_integration.py +++ b/sdk/python/tests/test_agent_integration.py @@ -389,9 +389,9 @@ async def test_func() -> dict: agent.include_router(router) expected_id = f"{expected_segment}_test_func" - assert any( - r["id"] == expected_id for r in agent.reasoners - ), f"Failed for prefix '{prefix}': expected '{expected_id}'" + assert any(r["id"] == expected_id for r in agent.reasoners), ( + f"Failed for prefix '{prefix}': expected '{expected_id}'" + ) @pytest.mark.asyncio diff --git a/sdk/python/tests/test_agent_lifecycle_invariants.py b/sdk/python/tests/test_agent_lifecycle_invariants.py index d7d072d27..d2f718b14 100644 --- a/sdk/python/tests/test_agent_lifecycle_invariants.py +++ b/sdk/python/tests/test_agent_lifecycle_invariants.py @@ -9,11 +9,13 @@ - Discovery via {'path': '/discover'} returns flat dict: node_id, version, reasoners, skills - Each reasoner dict in discovery list has 'id' = function.__name__ """ + from __future__ import annotations def _make_agent(node_id: str = "test-lifecycle-node"): from agentfield.agent import Agent + return Agent(node_id=node_id, agentfield_server="http://localhost:8080") @@ -246,8 +248,12 @@ def stable_fn() -> dict: assert result1.get("node_id") == result2.get("node_id"), ( "INVARIANT VIOLATION: Discovery node_id changed between calls." ) - ids1 = {r.get("id") for r in result1.get("reasoners", []) if isinstance(r, dict)} - ids2 = {r.get("id") for r in result2.get("reasoners", []) if isinstance(r, dict)} + ids1 = { + r.get("id") for r in result1.get("reasoners", []) if isinstance(r, dict) + } + ids2 = { + r.get("id") for r in result2.get("reasoners", []) if isinstance(r, dict) + } assert ids1 == ids2, ( f"INVARIANT VIOLATION: Discovery reasoner IDs changed: {ids1} vs {ids2}" ) diff --git a/sdk/python/tests/test_agent_networking.py b/sdk/python/tests/test_agent_networking.py index a97348ab3..005fe4962 100644 --- a/sdk/python/tests/test_agent_networking.py +++ b/sdk/python/tests/test_agent_networking.py @@ -29,11 +29,22 @@ def json(self): def fake_get(url, headers=None, timeout=None): calls.append(url) parsed = urlparse(url) - if parsed.netloc == "169.254.169.254" and parsed.path == "/latest/meta-data/public-ipv4": + if ( + parsed.netloc == "169.254.169.254" + and parsed.path == "/latest/meta-data/public-ipv4" + ): return DummyResponse(200, "198.51.100.5") - if parsed.netloc == "metadata.google.internal" and parsed.path == "/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip": + if ( + parsed.netloc == "metadata.google.internal" + and parsed.path + == "/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" + ): return DummyResponse(200, "203.0.113.7") - if parsed.scheme == "https" and parsed.netloc == "api.ipify.org" and parsed.path in {"", "/"}: + if ( + parsed.scheme == "https" + and parsed.netloc == "api.ipify.org" + and parsed.path in {"", "/"} + ): return DummyResponse(200, "192.0.2.9") return DummyResponse(404, "") diff --git a/sdk/python/tests/test_agent_registry.py b/sdk/python/tests/test_agent_registry.py index c433a12a0..38cf0f0f5 100644 --- a/sdk/python/tests/test_agent_registry.py +++ b/sdk/python/tests/test_agent_registry.py @@ -6,6 +6,7 @@ import queue import threading + class DummyAgent: pass diff --git a/sdk/python/tests/test_agent_server.py b/sdk/python/tests/test_agent_server.py index 7e7dd847e..3ef708ee2 100644 --- a/sdk/python/tests/test_agent_server.py +++ b/sdk/python/tests/test_agent_server.py @@ -1,6 +1,7 @@ """ Tests for agentfield.agent_server — AgentServer route registration and utility methods. """ + from __future__ import annotations import asyncio @@ -152,9 +153,7 @@ async def existing_lifespan(app): app.lifecycle_events = events app.agentfield_handler = SimpleNamespace( start_heartbeat=lambda interval: events.append("heartbeat-start"), - setup_fast_lifecycle_signal_handlers=lambda: events.append( - "signal-handlers" - ), + setup_fast_lifecycle_signal_handlers=lambda: events.append("signal-handlers"), stop_heartbeat=lambda: events.append("heartbeat-stop"), send_enhanced_heartbeat=AsyncMock(return_value=True), enhanced_heartbeat_loop=AsyncMock(), @@ -170,7 +169,9 @@ async def exercise_lifespan(): asyncio.run(exercise_lifespan()) - monkeypatch.setattr("agentfield.connection_manager.ConnectionManager", _FakeConnectionManager) + monkeypatch.setattr( + "agentfield.connection_manager.ConnectionManager", _FakeConnectionManager + ) monkeypatch.setattr("agentfield.agent_server.uvicorn.run", fake_uvicorn_run) AgentServer(app).serve(port=8001) @@ -310,7 +311,9 @@ def cpu_percent(self): def num_threads(self): return 1 - with patch.dict(sys.modules, {"psutil": SimpleNamespace(Process=lambda: DummyProcess())}): + with patch.dict( + sys.modules, {"psutil": SimpleNamespace(Process=lambda: DummyProcess())} + ): resp = await _get(app, "/status") assert resp.json()["status"] == "stopping" @@ -465,7 +468,12 @@ def test_cert_none(self): def test_nonexistent_files(self, tmp_path): s = self._server(dev_mode=True) - assert s._validate_ssl_config(str(tmp_path / "nope.key"), str(tmp_path / "nope.crt")) is False + assert ( + s._validate_ssl_config( + str(tmp_path / "nope.key"), str(tmp_path / "nope.crt") + ) + is False + ) def test_valid_files(self, tmp_path): key = tmp_path / "server.key" @@ -530,8 +538,10 @@ async def test_logs_endpoint_disabled(): async def test_logs_endpoint_unauthorized(): app = make_agent_app() _setup_server(app) - with patch("agentfield.node_logs.logs_enabled", return_value=True), \ - patch("agentfield.node_logs.verify_internal_bearer", return_value=False): + with ( + patch("agentfield.node_logs.logs_enabled", return_value=True), + patch("agentfield.node_logs.verify_internal_bearer", return_value=False), + ): resp = await _get(app, "/agentfield/v1/logs") assert resp.status_code == 401 @@ -541,8 +551,10 @@ async def test_logs_endpoint_tail_too_large(monkeypatch): app = make_agent_app() _setup_server(app) monkeypatch.setenv("AGENTFIELD_LOG_MAX_TAIL_LINES", "100") - with patch("agentfield.node_logs.logs_enabled", return_value=True), \ - patch("agentfield.node_logs.verify_internal_bearer", return_value=True): + with ( + patch("agentfield.node_logs.logs_enabled", return_value=True), + patch("agentfield.node_logs.verify_internal_bearer", return_value=True), + ): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://test" ) as client: @@ -558,9 +570,11 @@ async def test_logs_endpoint_success(): async def fake_iter(tail, since, follow): yield '{"line": 1}\n' - with patch("agentfield.node_logs.logs_enabled", return_value=True), \ - patch("agentfield.node_logs.verify_internal_bearer", return_value=True), \ - patch("agentfield.node_logs.iter_tail_ndjson", side_effect=fake_iter): + with ( + patch("agentfield.node_logs.logs_enabled", return_value=True), + patch("agentfield.node_logs.verify_internal_bearer", return_value=True), + patch("agentfield.node_logs.iter_tail_ndjson", side_effect=fake_iter), + ): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://test" ) as client: @@ -635,8 +649,7 @@ async def hung_coroutine(): joined = "\n".join(body["tasks"]) assert "simulated-hung-llm-call" in joined, ( "/debug/tasks must surface tasks suspended on Futures so we can " - "diagnose deadlocks in production. Found tasks: " - + joined[:500] + "diagnose deadlocks in production. Found tasks: " + joined[:500] ) finally: pending_event.set() diff --git a/sdk/python/tests/test_agent_server_extended.py b/sdk/python/tests/test_agent_server_extended.py index f0400a802..15b457951 100644 --- a/sdk/python/tests/test_agent_server_extended.py +++ b/sdk/python/tests/test_agent_server_extended.py @@ -246,7 +246,9 @@ def cpu_percent(self): def num_threads(self): return 1 - monkeypatch.setitem(sys.modules, "psutil", SimpleNamespace(Process=lambda: _FakeProcess())) + monkeypatch.setitem( + sys.modules, "psutil", SimpleNamespace(Process=lambda: _FakeProcess()) + ) resp = await _get(app, "/status") diff --git a/sdk/python/tests/test_agent_serverless.py b/sdk/python/tests/test_agent_serverless.py index 9886c165e..3b8ca366d 100644 --- a/sdk/python/tests/test_agent_serverless.py +++ b/sdk/python/tests/test_agent_serverless.py @@ -1,6 +1,7 @@ """ Tests for Agent.handle_serverless — serverless invocation path. """ + from __future__ import annotations from unittest.mock import patch @@ -16,9 +17,11 @@ def _make_agent() -> Agent: """Construct a minimal Agent without network side-effects.""" - with patch("agentfield.agent._detect_container_ip", return_value=None), \ - patch("agentfield.agent._detect_local_ip", return_value="127.0.0.1"), \ - patch("agentfield.agent._is_running_in_container", return_value=False): + with ( + patch("agentfield.agent._detect_container_ip", return_value=None), + patch("agentfield.agent._detect_local_ip", return_value="127.0.0.1"), + patch("agentfield.agent._is_running_in_container", return_value=False), + ): agent = Agent( node_id="test-agent", agentfield_server="http://localhost:8080", @@ -115,10 +118,12 @@ def greet(name: str) -> dict: # Patch getattr to find our function with patch.object(agent, "greet", greet, create=True): - result = agent.handle_serverless({ - "reasoner": "greet", - "input": {"name": "world"}, - }) + result = agent.handle_serverless( + { + "reasoner": "greet", + "input": {"name": "world"}, + } + ) assert result["statusCode"] == 200 assert result["body"]["greeting"] == "hello world" @@ -130,10 +135,12 @@ async def async_greet(name: str) -> dict: return {"greeting": f"async hello {name}"} with patch.object(agent, "async_greet", async_greet, create=True): - result = agent.handle_serverless({ - "reasoner": "async_greet", - "input": {"name": "world"}, - }) + result = agent.handle_serverless( + { + "reasoner": "async_greet", + "input": {"name": "world"}, + } + ) assert result["statusCode"] == 200 assert result["body"]["greeting"] == "async hello world" @@ -145,10 +152,12 @@ def bad_fn() -> dict: raise ValueError("something went wrong") with patch.object(agent, "bad_fn", bad_fn, create=True): - result = agent.handle_serverless({ - "reasoner": "bad_fn", - "input": {}, - }) + result = agent.handle_serverless( + { + "reasoner": "bad_fn", + "input": {}, + } + ) assert result["statusCode"] == 500 assert "something went wrong" in result["body"]["error"] @@ -167,10 +176,12 @@ def echo(msg: str) -> dict: return {"msg": msg} with patch.object(agent, "echo", echo, create=True): - result = agent.handle_serverless({ - "target": "echo", - "input": {"msg": "hi"}, - }) + result = agent.handle_serverless( + { + "target": "echo", + "input": {"msg": "hi"}, + } + ) assert result["statusCode"] == 200 @@ -181,10 +192,12 @@ def echo(msg: str) -> dict: return {"msg": msg} with patch.object(agent, "echo", echo, create=True): - result = agent.handle_serverless({ - "skill": "echo", - "input": {"msg": "test"}, - }) + result = agent.handle_serverless( + { + "skill": "echo", + "input": {"msg": "test"}, + } + ) assert result["statusCode"] == 200 @@ -195,10 +208,12 @@ def echo(msg: str) -> dict: return {"msg": msg} with patch.object(agent, "echo", echo, create=True): - result = agent.handle_serverless({ - "path": "/execute/echo", - "input": {"msg": "via-path"}, - }) + result = agent.handle_serverless( + { + "path": "/execute/echo", + "input": {"msg": "via-path"}, + } + ) assert result["statusCode"] == 200 @@ -209,14 +224,16 @@ def noop() -> dict: return {} with patch.object(agent, "noop", noop, create=True): - result = agent.handle_serverless({ - "reasoner": "noop", - "input": {}, - "execution_context": { - "execution_id": "custom-exec-id", - "run_id": "custom-run-id", - }, - }) + result = agent.handle_serverless( + { + "reasoner": "noop", + "input": {}, + "execution_context": { + "execution_id": "custom-exec-id", + "run_id": "custom-run-id", + }, + } + ) assert result["statusCode"] == 200 @@ -292,10 +309,12 @@ def test_registered_reasoner_invoked_via_serverless(self): def real_greet(name: str = "world") -> dict: return {"hello": name} - result = agent.handle_serverless({ - "reasoner": "real_greet", - "input": {"name": "test"}, - }) + result = agent.handle_serverless( + { + "reasoner": "real_greet", + "input": {"name": "test"}, + } + ) assert result["statusCode"] == 200 assert result["body"]["hello"] == "test" @@ -308,9 +327,11 @@ def test_registered_reasoner_not_found_returns_404(self): def exists() -> dict: return {} - result = agent.handle_serverless({ - "reasoner": "does_not_exist", - "input": {}, - }) + result = agent.handle_serverless( + { + "reasoner": "does_not_exist", + "input": {}, + } + ) assert result["statusCode"] == 404 diff --git a/sdk/python/tests/test_agent_workflow.py b/sdk/python/tests/test_agent_workflow.py index 3dd02d5c8..0ab839e87 100644 --- a/sdk/python/tests/test_agent_workflow.py +++ b/sdk/python/tests/test_agent_workflow.py @@ -348,6 +348,6 @@ async def parent_reasoner(execution_context: ExecutionContext = None): if entry[1] == "succeeded" and entry[0].endswith("parent_reasoner") ) - assert ( - child_complete_index < parent_complete_index - ), "Parent reasoner completion emitted before child finished" + assert child_complete_index < parent_complete_index, ( + "Parent reasoner completion emitted before child finished" + ) diff --git a/sdk/python/tests/test_agent_workflow_extended.py b/sdk/python/tests/test_agent_workflow_extended.py index 4c67ef528..c31d95bdf 100644 --- a/sdk/python/tests/test_agent_workflow_extended.py +++ b/sdk/python/tests/test_agent_workflow_extended.py @@ -45,7 +45,13 @@ def __init__(self): async def _async_request(self, method: str, url: str, **kwargs): self.calls.append({"method": method, "url": url, "kwargs": kwargs}) # Return a fake response with .json() - return SimpleNamespace(json=lambda: {"execution_id": "srv-exec-1", "workflow_id": "srv-wf-1", "run_id": "srv-run-1"}) + return SimpleNamespace( + json=lambda: { + "execution_id": "srv-exec-1", + "workflow_id": "srv-wf-1", + "run_id": "srv-run-1", + } + ) agent.client = _Client() return agent @@ -371,7 +377,9 @@ async def test_child_execution_inherits_parent_workflow_id(monkeypatch): child_contexts: List[ExecutionContext] = [] - async def noop_start(execution_id, context, reasoner_name, input_data, parent_execution_id=None): + async def noop_start( + execution_id, context, reasoner_name, input_data, parent_execution_id=None + ): child_contexts.append(context) async def noop_complete(*args, **kwargs): diff --git a/sdk/python/tests/test_approval.py b/sdk/python/tests/test_approval.py index 9146c104f..57a2b0d16 100644 --- a/sdk/python/tests/test_approval.py +++ b/sdk/python/tests/test_approval.py @@ -23,7 +23,9 @@ class FakeResponse: - def __init__(self, status_code: int = 200, payload: dict | None = None, text: str = ""): + def __init__( + self, status_code: int = 200, payload: dict | None = None, text: str = "" + ): self.status_code = status_code self._payload = payload or {} self.text = text or "{}" @@ -40,7 +42,9 @@ def client() -> AgentFieldClient: @pytest.mark.asyncio -async def test_request_approval_returns_typed_response_and_payload(client: AgentFieldClient): +async def test_request_approval_returns_typed_response_and_payload( + client: AgentFieldClient, +): http_client = SimpleNamespace( post=AsyncMock( return_value=FakeResponse( @@ -88,7 +92,9 @@ async def test_request_approval_wraps_transport_errors(client: AgentFieldClient) @pytest.mark.asyncio async def test_request_approval_raises_on_http_error(client: AgentFieldClient): http_client = SimpleNamespace( - post=AsyncMock(return_value=FakeResponse(status_code=404, text='{"error":"missing"}')) + post=AsyncMock( + return_value=FakeResponse(status_code=404, text='{"error":"missing"}') + ) ) client.get_async_http_client = AsyncMock(return_value=http_client) @@ -132,14 +138,18 @@ async def test_get_approval_status_wraps_transport_errors(client: AgentFieldClie http_client = SimpleNamespace(get=AsyncMock(side_effect=RuntimeError("boom"))) client.get_async_http_client = AsyncMock(return_value=http_client) - with pytest.raises(AgentFieldClientError, match="Failed to get approval status: boom"): + with pytest.raises( + AgentFieldClientError, match="Failed to get approval status: boom" + ): await client.get_approval_status(EXECUTION_ID) @pytest.mark.asyncio async def test_get_approval_status_raises_on_http_error(client: AgentFieldClient): http_client = SimpleNamespace( - get=AsyncMock(return_value=FakeResponse(status_code=500, text='{"error":"internal"}')) + get=AsyncMock( + return_value=FakeResponse(status_code=500, text='{"error":"internal"}') + ) ) client.get_async_http_client = AsyncMock(return_value=http_client) @@ -152,7 +162,9 @@ async def test_wait_for_approval_returns_first_resolved_status( client: AgentFieldClient, monkeypatch: pytest.MonkeyPatch ): pending = ApprovalStatusResponse(status="pending") - approved = ApprovalStatusResponse(status="approved", response={"decision": "approved"}) + approved = ApprovalStatusResponse( + status="approved", response={"decision": "approved"} + ) client.get_approval_status = AsyncMock(side_effect=[pending, approved]) async def no_sleep(_: float) -> None: @@ -160,7 +172,9 @@ async def no_sleep(_: float) -> None: monkeypatch.setattr("agentfield.client.asyncio.sleep", no_sleep) - result = await client.wait_for_approval(EXECUTION_ID, poll_interval=0.01, max_interval=0.02) + result = await client.wait_for_approval( + EXECUTION_ID, poll_interval=0.01, max_interval=0.02 + ) assert result.status == "approved" assert client.get_approval_status.await_count == 2 @@ -170,7 +184,9 @@ async def no_sleep(_: float) -> None: async def test_wait_for_approval_retries_transient_client_errors( client: AgentFieldClient, monkeypatch: pytest.MonkeyPatch ): - approved = ApprovalStatusResponse(status="approved", response={"decision": "approved"}) + approved = ApprovalStatusResponse( + status="approved", response={"decision": "approved"} + ) client.get_approval_status = AsyncMock( side_effect=[AgentFieldClientError("transient"), approved] ) @@ -180,7 +196,9 @@ async def no_sleep(_: float) -> None: monkeypatch.setattr("agentfield.client.asyncio.sleep", no_sleep) - result = await client.wait_for_approval(EXECUTION_ID, poll_interval=0.01, max_interval=0.02) + result = await client.wait_for_approval( + EXECUTION_ID, poll_interval=0.01, max_interval=0.02 + ) assert result.status == "approved" assert client.get_approval_status.await_count == 2 @@ -192,7 +210,9 @@ async def test_wait_for_approval_times_out( ): import agentfield.client as client_module - client.get_approval_status = AsyncMock(return_value=ApprovalStatusResponse(status="pending")) + client.get_approval_status = AsyncMock( + return_value=ApprovalStatusResponse(status="pending") + ) async def no_sleep(_: float) -> None: return None @@ -213,7 +233,9 @@ async def no_sleep(_: float) -> None: @pytest.mark.asyncio async def test_batch_check_statuses_uses_batched_path(client: AgentFieldClient): manager = SimpleNamespace( - get_execution_status=AsyncMock(side_effect=[{"status": "queued"}, {"status": "running"}]) + get_execution_status=AsyncMock( + side_effect=[{"status": "queued"}, {"status": "running"}] + ) ) client._get_async_execution_manager = AsyncMock(return_value=manager) client.async_config.enable_async_execution = True @@ -228,7 +250,9 @@ async def test_batch_check_statuses_uses_batched_path(client: AgentFieldClient): @pytest.mark.asyncio async def test_batch_check_statuses_uses_individual_path(client: AgentFieldClient): - manager = SimpleNamespace(get_execution_status=AsyncMock(return_value={"status": "done"})) + manager = SimpleNamespace( + get_execution_status=AsyncMock(return_value={"status": "done"}) + ) client._get_async_execution_manager = AsyncMock(return_value=manager) client.async_config.enable_async_execution = True client.async_config.enable_batch_polling = False @@ -241,7 +265,9 @@ async def test_batch_check_statuses_uses_individual_path(client: AgentFieldClien @pytest.mark.asyncio async def test_list_async_executions_normalizes_status_filter(client: AgentFieldClient): - manager = SimpleNamespace(list_executions=AsyncMock(return_value=[{"id": "exec-1"}])) + manager = SimpleNamespace( + list_executions=AsyncMock(return_value=[{"id": "exec-1"}]) + ) client._get_async_execution_manager = AsyncMock(return_value=manager) client.async_config.enable_async_execution = True @@ -252,7 +278,9 @@ async def test_list_async_executions_normalizes_status_filter(client: AgentField @pytest.mark.asyncio -async def test_list_async_executions_returns_empty_for_invalid_status(client: AgentFieldClient): +async def test_list_async_executions_returns_empty_for_invalid_status( + client: AgentFieldClient, +): manager = SimpleNamespace(list_executions=AsyncMock()) client._get_async_execution_manager = AsyncMock(return_value=manager) client.async_config.enable_async_execution = True @@ -264,7 +292,9 @@ async def test_list_async_executions_returns_empty_for_invalid_status(client: Ag @pytest.mark.asyncio -async def test_close_async_execution_manager_stops_and_clears_manager(client: AgentFieldClient): +async def test_close_async_execution_manager_stops_and_clears_manager( + client: AgentFieldClient, +): manager = SimpleNamespace(stop=AsyncMock()) client._async_execution_manager = manager diff --git a/sdk/python/tests/test_async_execution.py b/sdk/python/tests/test_async_execution.py index 5f87a35d6..05cdbdcf9 100644 --- a/sdk/python/tests/test_async_execution.py +++ b/sdk/python/tests/test_async_execution.py @@ -774,9 +774,7 @@ async def _drive_polling(): poller = asyncio.create_task(_drive_polling()) with pytest.raises(ExecutionCancelledError): - await manager.wait_for_result( - exec_id, timeout=5.0, pause_clock=parent_clock - ) + await manager.wait_for_result(exec_id, timeout=5.0, pause_clock=parent_clock) await poller # Pause must be closed: total_paused stops accumulating once the wait @@ -945,8 +943,7 @@ def total_paused(self) -> float: await manager._poll_active_executions() assert state.status == ExecutionStatus.WAITING, ( - f"polling task must not flip a paused execution to TIMEOUT, " - f"got {state.status}" + f"polling task must not flip a paused execution to TIMEOUT, got {state.status}" ) diff --git a/sdk/python/tests/test_async_execution_manager_final90.py b/sdk/python/tests/test_async_execution_manager_final90.py index 1af779334..b551c9994 100644 --- a/sdk/python/tests/test_async_execution_manager_final90.py +++ b/sdk/python/tests/test_async_execution_manager_final90.py @@ -78,7 +78,9 @@ def fake_create_task(coro): created_tasks.append(task) return task - monkeypatch.setattr("agentfield.async_execution_manager.asyncio.create_task", fake_create_task) + monkeypatch.setattr( + "agentfield.async_execution_manager.asyncio.create_task", fake_create_task + ) manager.config.enable_performance_logging = True manager.config.enable_event_stream = True @@ -101,7 +103,9 @@ def fake_create_task(coro): @pytest.mark.asyncio -async def test_wait_for_result_handles_success_failure_cancel_timeout(manager, monkeypatch): +async def test_wait_for_result_handles_success_failure_cancel_timeout( + manager, monkeypatch +): success = ExecutionState("success", "node.skill", {}) success.set_result({"ok": True}) @@ -124,7 +128,9 @@ async def test_wait_for_result_handles_success_failure_cancel_timeout(manager, m ) assert await manager.wait_for_result("success") == {"ok": True} - manager.result_cache.set_execution_result.assert_called_with("success", {"ok": True}) + manager.result_cache.set_execution_result.assert_called_with( + "success", {"ok": True} + ) with pytest.raises(AgentFieldClientError, match="boom"): await manager.wait_for_result("failed") @@ -231,12 +237,18 @@ def fake_create_task(coro): coro.close() return _DummyTask() - monkeypatch.setattr("agentfield.async_execution_manager.asyncio.create_task", fake_create_task) + monkeypatch.setattr( + "agentfield.async_execution_manager.asyncio.create_task", fake_create_task + ) - await manager._handle_event_stream_payload({"execution_id": "running", "status": "running"}) + await manager._handle_event_stream_payload( + {"execution_id": "running", "status": "running"} + ) assert running.status == ExecutionStatus.RUNNING - await manager._handle_event_stream_payload({"executionId": "running", "status": "succeeded"}) + await manager._handle_event_stream_payload( + {"executionId": "running", "status": "succeeded"} + ) assert running.status == ExecutionStatus.SUCCEEDED assert create_task_calls @@ -252,7 +264,9 @@ async def test_process_poll_response_metrics_and_circuit_breaker(manager, monkey assert manager.metrics.polling_metrics.timeout_polls == 1 assert execution.current_poll_interval > old_interval - await manager._process_poll_response(execution, _Response({"status": "running"}), 0.2) + await manager._process_poll_response( + execution, _Response({"status": "running"}), 0.2 + ) update_mock.assert_awaited_once() assert manager.metrics.polling_metrics.successful_polls == 1 @@ -262,9 +276,11 @@ async def test_process_poll_response_metrics_and_circuit_breaker(manager, monkey monkeypatch.setattr( "agentfield.async_execution_manager.time.time", - lambda: manager._circuit_breaker_last_failure - + manager.config.circuit_breaker_recovery_timeout - + 1, + lambda: ( + manager._circuit_breaker_last_failure + + manager.config.circuit_breaker_recovery_timeout + + 1 + ), ) assert manager._is_circuit_breaker_open() is False assert "success_rate" in repr(manager) diff --git a/sdk/python/tests/test_async_execution_manager_paths.py b/sdk/python/tests/test_async_execution_manager_paths.py index f1fb7a672..2727258ae 100644 --- a/sdk/python/tests/test_async_execution_manager_paths.py +++ b/sdk/python/tests/test_async_execution_manager_paths.py @@ -131,6 +131,7 @@ async def get_session(self): call = session_post.await_args assert call.args[0] == "http://example/api/v1/execute/async/node.reasoner" import json as json_module + assert json_module.loads(call.kwargs["data"]) == {"input": {"foo": "bar"}} diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index a7629123b..f83077491 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -375,21 +375,25 @@ def on_request(method, url, **kwargs): # UTC timestamp helper tests (added per code review feedback) # --------------------------------------------------------------------------- + class TestUtcNowIso: """Tests for the _utc_now_iso helper used in registration payloads.""" def test_returns_string(self): from agentfield.client import _utc_now_iso + assert isinstance(_utc_now_iso(), str) def test_ends_with_z(self): from agentfield.client import _utc_now_iso + assert _utc_now_iso().endswith("Z") def test_is_valid_iso_format(self): """Timestamp must parse back as UTC-aware datetime without error.""" import datetime from agentfield.client import _utc_now_iso + ts = _utc_now_iso() # Remove trailing Z and parse parsed = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")) @@ -399,6 +403,7 @@ def test_is_valid_iso_format(self): def test_no_double_offset(self): """Must not produce invalid strings like 2026-01-01T00:00:00+00:00Z.""" from agentfield.client import _utc_now_iso + ts = _utc_now_iso() assert "+00:00Z" not in ts assert ts.count("Z") == 1 @@ -406,6 +411,7 @@ def test_no_double_offset(self): def test_millisecond_precision(self): """Timestamp should have millisecond precision (3 decimal digits).""" from agentfield.client import _utc_now_iso + ts = _utc_now_iso() # Format: 2026-04-05T01:23:45.678Z body = ts[:-1] # strip Z diff --git a/sdk/python/tests/test_client_auth.py b/sdk/python/tests/test_client_auth.py index 1f3d38d05..0f4e72abe 100644 --- a/sdk/python/tests/test_client_auth.py +++ b/sdk/python/tests/test_client_auth.py @@ -303,7 +303,9 @@ def fake_get(url, headers=None, timeout=None): monkeypatch.setattr(client_mod.requests, "post", fake_post) monkeypatch.setattr(client_mod.requests, "get", fake_get) - client = AgentFieldClient(base_url="http://example.com", api_key="configured-key") + client = AgentFieldClient( + base_url="http://example.com", api_key="configured-key" + ) # Try to override with custom header client.execute_sync( "node.reasoner", @@ -332,7 +334,9 @@ def on_request(method, url, **kwargs): from agentfield.types import AgentStatus, HeartbeatData - client = AgentFieldClient(base_url="http://example.com", api_key="heartbeat-key") + client = AgentFieldClient( + base_url="http://example.com", api_key="heartbeat-key" + ) heartbeat = HeartbeatData(status=AgentStatus.READY, timestamp="now") result = await client.send_enhanced_heartbeat("node-1", heartbeat) @@ -348,9 +352,11 @@ def test_heartbeat_sync_includes_api_key(self, monkeypatch): def fake_post(url, json=None, headers=None, timeout=None): captured["headers"] = headers + class Resp: def raise_for_status(self): pass + return Resp() import agentfield.client as client_mod @@ -359,7 +365,9 @@ def raise_for_status(self): from agentfield.types import AgentStatus, HeartbeatData - client = AgentFieldClient(base_url="http://example.com", api_key="heartbeat-key") + client = AgentFieldClient( + base_url="http://example.com", api_key="heartbeat-key" + ) heartbeat = HeartbeatData(status=AgentStatus.READY, timestamp="now") result = client.send_enhanced_heartbeat_sync("node-1", heartbeat) @@ -491,5 +499,7 @@ def test_unattached_router_raises_error(self): router = AgentRouter(prefix="/test") - with pytest.raises(AgentFieldClientError, match="Router not attached to an agent"): + with pytest.raises( + AgentFieldClientError, match="Router not attached to an agent" + ): _ = router.api_key diff --git a/sdk/python/tests/test_client_bigfiles_coverage.py b/sdk/python/tests/test_client_bigfiles_coverage.py index 402fbda7e..e7fa4188f 100644 --- a/sdk/python/tests/test_client_bigfiles_coverage.py +++ b/sdk/python/tests/test_client_bigfiles_coverage.py @@ -49,7 +49,11 @@ def test_get_headers_with_context_uses_fallback_when_context_raises(client): def test_maybe_update_event_stream_headers_uses_context_and_manager(client): calls = [] client._current_workflow_context = SimpleNamespace( - to_headers=lambda: {"Authorization": "Bearer token", "X-Trace": "abc", "Ignore": "nope"} + to_headers=lambda: { + "Authorization": "Bearer token", + "X-Trace": "abc", + "Ignore": "nope", + } ) client._async_execution_manager = SimpleNamespace( set_event_stream_headers=lambda headers: calls.append(headers) @@ -120,7 +124,9 @@ def test_parse_submission_requires_identifiers(client): @pytest.mark.asyncio -async def test_submit_execution_async_raises_execute_error_and_formats_body(monkeypatch, client): +async def test_submit_execution_async_raises_execute_error_and_formats_body( + monkeypatch, client +): async def fake_request(method, url, **kwargs): assert kwargs["content"] == b'{"input":{"value":1}}' return DummyResponse(status_code=400, payload={"error": "bad input"}) @@ -145,16 +151,22 @@ def test_await_execution_sync_uses_cache_and_failure_payload(monkeypatch, client target="node.reasoner", status="queued", ) - client._result_cache = SimpleNamespace(get_execution_result=lambda execution_id: {"cached": True}) + client._result_cache = SimpleNamespace( + get_execution_result=lambda execution_id: {"cached": True} + ) cached = client._await_execution_sync(submission, {"X-Run-ID": "run-1"}) assert cached["status"] == "succeeded" assert cached["result"] == {"cached": True} - client._result_cache = SimpleNamespace(get_execution_result=lambda execution_id: None) + client._result_cache = SimpleNamespace( + get_execution_result=lambda execution_id: None + ) def fake_get(url, headers=None, timeout=None): - return DummyResponse(status_code=200, payload={"status": "FAILED", "error": "nope"}) + return DummyResponse( + status_code=200, payload={"status": "FAILED", "error": "nope"} + ) monkeypatch.setattr("agentfield.client.requests.get", fake_get) @@ -171,7 +183,9 @@ async def test_await_execution_async_times_out(monkeypatch, client): target="node.reasoner", status="queued", ) - client._result_cache = SimpleNamespace(get_execution_result=lambda execution_id: None) + client._result_cache = SimpleNamespace( + get_execution_result=lambda execution_id: None + ) client.async_config.initial_poll_interval = 0.01 client.async_config.max_execution_timeout = 0.05 @@ -224,7 +238,9 @@ def test_format_execution_result_and_build_response_for_failure(client): @pytest.mark.asyncio -async def test_execute_async_falls_back_to_sync_and_skips_auth_errors(monkeypatch, client): +async def test_execute_async_falls_back_to_sync_and_skips_auth_errors( + monkeypatch, client +): client.async_config.fallback_to_sync = True client._get_async_execution_manager = AsyncMock(side_effect=RuntimeError("down")) client.execute = AsyncMock(return_value={"status": "succeeded"}) @@ -236,7 +252,9 @@ async def test_execute_async_falls_back_to_sync_and_skips_auth_errors(monkeypatc assert execution_id == synthetic client.execute.assert_awaited_once() - client._get_async_execution_manager = AsyncMock(side_effect=ExecuteError(401, "unauthorized")) + client._get_async_execution_manager = AsyncMock( + side_effect=ExecuteError(401, "unauthorized") + ) client.execute.reset_mock() with pytest.raises(ExecuteError): diff --git a/sdk/python/tests/test_client_coverage_additions.py b/sdk/python/tests/test_client_coverage_additions.py index 946331679..3096ff402 100644 --- a/sdk/python/tests/test_client_coverage_additions.py +++ b/sdk/python/tests/test_client_coverage_additions.py @@ -36,7 +36,9 @@ def client(): @pytest.mark.asyncio -async def test_get_async_http_client_falls_back_for_simple_test_double(monkeypatch, client): +async def test_get_async_http_client_falls_back_for_simple_test_double( + monkeypatch, client +): class FlexibleAsyncClient(DummyAsyncClient): def __init__(self, **kwargs): if kwargs: @@ -49,7 +51,9 @@ class HttpxStub: Timeout = None monkeypatch.setattr("agentfield.client.httpx", None) - monkeypatch.setattr("agentfield.client._ensure_httpx", lambda force_reload=False: HttpxStub) + monkeypatch.setattr( + "agentfield.client._ensure_httpx", lambda force_reload=False: HttpxStub + ) async_client = await client.get_async_http_client() @@ -120,17 +124,23 @@ def test_submit_execution_sync_raises_execute_error_with_message(monkeypatch, cl is_configured=True, sign_headers=signer, ) - monkeypatch.setattr("agentfield.client.requests.post", lambda *args, **kwargs: response) + monkeypatch.setattr( + "agentfield.client.requests.post", lambda *args, **kwargs: response + ) with pytest.raises(ExecuteError) as excinfo: - client._submit_execution_sync("agent.reasoner", {"value": 1}, {"X-Run-ID": "run-1"}) + client._submit_execution_sync( + "agent.reasoner", {"value": 1}, {"X-Run-ID": "run-1"} + ) assert excinfo.value.status_code == 422 assert "invalid input" in str(excinfo.value) signer.assert_called_once() -def test_format_execution_result_caches_success_and_derives_node_id(monkeypatch, client): +def test_format_execution_result_caches_success_and_derives_node_id( + monkeypatch, client +): submission = _Submission( execution_id="exec-1", run_id="run-1", @@ -175,7 +185,9 @@ def request(self, method, url, **kwargs): session = Session() debug = Mock() error = Mock() - monkeypatch.setattr(AgentFieldClient, "_get_sync_session", classmethod(lambda cls: session)) + monkeypatch.setattr( + AgentFieldClient, "_get_sync_session", classmethod(lambda cls: session) + ) monkeypatch.setattr("agentfield.client.logger.debug", debug) monkeypatch.setattr("agentfield.client.logger.error", error) diff --git a/sdk/python/tests/test_client_execution_vc_payload.py b/sdk/python/tests/test_client_execution_vc_payload.py index 294677725..c42e3115c 100644 --- a/sdk/python/tests/test_client_execution_vc_payload.py +++ b/sdk/python/tests/test_client_execution_vc_payload.py @@ -79,7 +79,9 @@ def test_vc_generator_includes_parent_vc_id_in_payload(): # Verify parent_vc_id is included in execution_context assert "parent_vc_id" in exec_ctx_payload - assert exec_ctx_payload["parent_vc_id"] == "vc_trigger_webhook_stripe_payment_123" + assert ( + exec_ctx_payload["parent_vc_id"] == "vc_trigger_webhook_stripe_payment_123" + ) @pytest.mark.unit diff --git a/sdk/python/tests/test_client_laser_push.py b/sdk/python/tests/test_client_laser_push.py index c8ea05a79..65fa8688e 100644 --- a/sdk/python/tests/test_client_laser_push.py +++ b/sdk/python/tests/test_client_laser_push.py @@ -25,7 +25,9 @@ class DummySyncResponse: - def __init__(self, payload=None, status_code=200, text="{}", content=None, headers=None): + def __init__( + self, payload=None, status_code=200, text="{}", content=None, headers=None + ): self._payload = {} if payload is None else payload self.status_code = status_code self.text = text @@ -79,7 +81,11 @@ def test_helper_properties_and_httpx_loader(monkeypatch): monkeypatch.setattr(client_mod, "httpx", sentinel) assert client_mod._ensure_httpx() is sentinel - monkeypatch.setattr(client_mod.importlib, "import_module", lambda name: (_ for _ in ()).throw(ImportError("nope"))) + monkeypatch.setattr( + client_mod.importlib, + "import_module", + lambda name: (_ for _ in ()).throw(ImportError("nope")), + ) monkeypatch.setattr(client_mod, "httpx", None) assert client_mod._ensure_httpx(force_reload=True) is None @@ -194,7 +200,9 @@ def fake_post(url, data=None, headers=None, timeout=None): ) with pytest.raises(AgentFieldClientError, match="Failed to submit execution"): - client._submit_execution_sync("node.reasoner", {"value": 1}, {"X-Run-ID": "run-1"}) + client._submit_execution_sync( + "node.reasoner", {"value": 1}, {"X-Run-ID": "run-1"} + ) def test_submit_execution_sync_raises_execute_error(monkeypatch, client): @@ -209,7 +217,9 @@ def test_submit_execution_sync_raises_execute_error(monkeypatch, client): ) with pytest.raises(ExecuteError) as excinfo: - client._submit_execution_sync("node.reasoner", {"value": 1}, {"X-Run-ID": "run-1"}) + client._submit_execution_sync( + "node.reasoner", {"value": 1}, {"X-Run-ID": "run-1"} + ) assert excinfo.value.status_code == 404 assert "bad target" in str(excinfo.value) @@ -236,7 +246,9 @@ def __init__(self, **kwargs): ) fallback_instance = FallbackAsyncClient() module.AsyncClient = lambda **kwargs: (_ for _ in ()).throw(TypeError("no kwargs")) - module.AsyncClient = lambda *args, **kwargs: (_ for _ in ()).throw(TypeError("no kwargs")) + module.AsyncClient = lambda *args, **kwargs: (_ for _ in ()).throw( + TypeError("no kwargs") + ) def fallback_ctor(*args, **kwargs): if kwargs: @@ -280,7 +292,9 @@ async def fake_to_thread(func, method, url, **kwargs): ) monkeypatch.setattr(client_mod, "_to_thread", fake_to_thread) - result = await client._async_request("GET", "http://example.test/ping", headers={"X-Test": "1"}) + result = await client._async_request( + "GET", "http://example.test/ping", headers={"X-Test": "1"} + ) assert result == {"ok": True} assert calls["headers"]["X-API-Key"] == "secret-key" @@ -300,11 +314,15 @@ def request(self, method, url, **kwargs): headers={"Content-Length": "5000"}, ) - monkeypatch.setattr(client_mod.AgentFieldClient, "_get_sync_session", lambda: DummySession()) + monkeypatch.setattr( + client_mod.AgentFieldClient, "_get_sync_session", lambda: DummySession() + ) monkeypatch.setattr(client_mod.logger, "error", logged["error"].append) monkeypatch.setattr(client_mod.logger, "debug", logged["debug"].append) - response = client._sync_request("POST", "http://example.test/submit", json={"value": 1}) + response = client._sync_request( + "POST", "http://example.test/submit", json={"value": 1} + ) assert response.status_code == 200 assert any("RESPONSE_TRUNCATION" in msg for msg in logged["error"]) @@ -407,7 +425,9 @@ async def request(self, method, url, **kwargs): captured["headers"] = kwargs["headers"] return {"ok": True} - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=FakeAsyncClient())) + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=FakeAsyncClient()) + ) result = await client._async_request("POST", "http://example.test/path") assert result == {"ok": True} @@ -415,11 +435,21 @@ async def request(self, method, url, **kwargs): @pytest.mark.asyncio -async def test_register_agent_with_status_parse_failures_and_logging(monkeypatch, client): - monkeypatch.setattr(client_mod.importlib, "import_module", lambda name: SimpleNamespace(__version__="9.9.9")) +async def test_register_agent_with_status_parse_failures_and_logging( + monkeypatch, client +): + monkeypatch.setattr( + client_mod.importlib, + "import_module", + lambda name: SimpleNamespace(__version__="9.9.9"), + ) logged = {"error": [], "debug": []} - monkeypatch.setattr(client_mod.logger, "error", lambda *args: logged["error"].append(args)) - monkeypatch.setattr(client_mod.logger, "debug", lambda *args: logged["debug"].append(args)) + monkeypatch.setattr( + client_mod.logger, "error", lambda *args: logged["error"].append(args) + ) + monkeypatch.setattr( + client_mod.logger, "debug", lambda *args: logged["debug"].append(args) + ) class BadJsonResponse(DummySyncResponse): def json(self): @@ -428,9 +458,13 @@ def json(self): monkeypatch.setattr( client, "_async_request", - AsyncMock(return_value=BadJsonResponse(status_code=500, text="broken", content=b"1")), + AsyncMock( + return_value=BadJsonResponse(status_code=500, text="broken", content=b"1") + ), + ) + failed, payload = await client.register_agent_with_status( + "node-5", [], [], "http://agent.test" ) - failed, payload = await client.register_agent_with_status("node-5", [], [], "http://agent.test") failed2, payload2 = await client.register_agent_with_status( "node-5", [], [], "http://agent.test", suppress_errors=True ) @@ -440,7 +474,9 @@ def json(self): assert logged["error"] assert logged["debug"] - monkeypatch.setattr(client, "_async_request", AsyncMock(side_effect=RuntimeError("boom"))) + monkeypatch.setattr( + client, "_async_request", AsyncMock(side_effect=RuntimeError("boom")) + ) failed3, payload3 = await client.register_agent_with_status( "node-6", [], [], "http://agent.test", suppress_errors=True ) @@ -476,7 +512,9 @@ async def submit_execution(self, **kwargs): monkeypatch.setattr(client_mod, "AsyncExecutionManager", FakeManager) manager = await client._get_async_execution_manager() - execution_id = await client.execute_async("node.reasoner", {"value": 1}, timeout=2.0) + execution_id = await client.execute_async( + "node.reasoner", {"value": 1}, timeout=2.0 + ) assert manager.kwargs["auth_headers"] == {"X-API-Key": "secret-key"} assert propagated[0] == "started" @@ -517,20 +555,28 @@ async def test_async_wrapper_disabled_and_error_branches(monkeypatch, client): wait_for_result=AsyncMock(side_effect=AgentFieldClientError("wait me")), cancel_execution=AsyncMock(side_effect=AgentFieldClientError("cancel me")), list_executions=AsyncMock(side_effect=RuntimeError("list boom")), - cleanup_completed_executions=AsyncMock(side_effect=RuntimeError("cleanup boom")), + cleanup_completed_executions=AsyncMock( + side_effect=RuntimeError("cleanup boom") + ), get_metrics=lambda: (_ for _ in ()).throw(RuntimeError("metrics boom")), stop=AsyncMock(side_effect=RuntimeError("stop boom")), set_event_stream_headers=lambda headers: None, ) client._async_execution_manager = manager - monkeypatch.setattr(client, "_get_async_execution_manager", AsyncMock(return_value=manager)) + monkeypatch.setattr( + client, "_get_async_execution_manager", AsyncMock(return_value=manager) + ) - with pytest.raises(AgentFieldClientError, match="Async execution failed for target"): + with pytest.raises( + AgentFieldClientError, match="Async execution failed for target" + ): await client.execute_async("node.reasoner", {"value": 1}) with pytest.raises(AgentFieldClientError, match="retry me"): await client.poll_execution_status("exec-2") manager.get_execution_status = AsyncMock(side_effect=RuntimeError("batch boom")) - with pytest.raises(AgentFieldClientError, match="Failed to batch check execution statuses"): + with pytest.raises( + AgentFieldClientError, match="Failed to batch check execution statuses" + ): await client.batch_check_statuses(["exec-2"]) with pytest.raises(AgentFieldClientError, match="wait me"): await client.wait_for_execution_result("exec-2") @@ -538,9 +584,13 @@ async def test_async_wrapper_disabled_and_error_branches(monkeypatch, client): await client.cancel_async_execution("exec-2") with pytest.raises(AgentFieldClientError, match="Failed to list async executions"): await client.list_async_executions("running") - with pytest.raises(AgentFieldClientError, match="Failed to get async execution metrics"): + with pytest.raises( + AgentFieldClientError, match="Failed to get async execution metrics" + ): await client.get_async_execution_metrics() - with pytest.raises(AgentFieldClientError, match="Failed to cleanup async executions"): + with pytest.raises( + AgentFieldClientError, match="Failed to cleanup async executions" + ): await client.cleanup_async_executions() with pytest.raises(RuntimeError, match="stop boom"): await client.close_async_execution_manager() @@ -555,17 +605,27 @@ async def post(self, *args, **kwargs): async def get(self, *args, **kwargs): raise RuntimeError("get boom") - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=RaisingClient())) + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=RaisingClient()) + ) - with pytest.raises(AgentFieldClientError, match="Failed to request approval: post boom"): + with pytest.raises( + AgentFieldClientError, match="Failed to request approval: post boom" + ): await client.request_approval("exec-9", "apr-9") - with pytest.raises(AgentFieldClientError, match="Failed to get approval status: get boom"): + with pytest.raises( + AgentFieldClientError, match="Failed to get approval status: get boom" + ): await client.get_approval_status("exec-9") @pytest.mark.asyncio async def test_register_agent_and_status_variants(monkeypatch, client): - monkeypatch.setattr(client_mod.importlib, "import_module", lambda name: SimpleNamespace(__version__="9.9.9")) + monkeypatch.setattr( + client_mod.importlib, + "import_module", + lambda name: SimpleNamespace(__version__="9.9.9"), + ) captured = [] @@ -600,17 +660,23 @@ async def fake_async_request(method, url, **kwargs): assert ok is True and payload == {"ok": True} assert ok2 is True and payload2 == {"ok": True} assert captured[0]["metadata"]["custom"]["team"] == "sdk" - assert captured[0]["metadata"]["custom"]["vc_generation"] == {"issuer": "did:web:test"} + assert captured[0]["metadata"]["custom"]["vc_generation"] == { + "issuer": "did:web:test" + } assert captured[0]["callback_discovery"] == {"public_url": "http://agent.test"} assert captured[0]["proposed_tags"] == ["blue"] assert captured[1]["lifecycle_status"] == "ready" assert captured[1]["communication_config"]["heartbeat_interval"] == "2s" async def non_201(*args, **kwargs): - return DummySyncResponse({"error": "bad"}, status_code=500, content=b'{"error":"bad"}') + return DummySyncResponse( + {"error": "bad"}, status_code=500, content=b'{"error":"bad"}' + ) monkeypatch.setattr(client, "_async_request", non_201) - failed, failure_payload = await client.register_agent("node-3", [], [], "http://agent.test") + failed, failure_payload = await client.register_agent( + "node-3", [], [], "http://agent.test" + ) failed2, failure_payload2 = await client.register_agent_with_status( "node-3", [], [], "http://agent.test", suppress_errors=True ) @@ -618,9 +684,15 @@ async def non_201(*args, **kwargs): assert failed is False and failure_payload == {"error": "bad"} assert failed2 is False and failure_payload2 == {"error": "bad"} - monkeypatch.setattr(client, "_async_request", AsyncMock(side_effect=RuntimeError("boom"))) - failed3, failure_payload3 = await client.register_agent("node-4", [], [], "http://agent.test") - failed4, failure_payload4 = await client.register_agent_with_status("node-4", [], [], "http://agent.test") + monkeypatch.setattr( + client, "_async_request", AsyncMock(side_effect=RuntimeError("boom")) + ) + failed3, failure_payload3 = await client.register_agent( + "node-4", [], [], "http://agent.test" + ) + failed4, failure_payload4 = await client.register_agent_with_status( + "node-4", [], [], "http://agent.test" + ) assert (failed3, failure_payload3) == (False, None) assert (failed4, failure_payload4) == (False, None) @@ -628,17 +700,29 @@ async def non_201(*args, **kwargs): @pytest.mark.asyncio async def test_heartbeat_and_shutdown_helpers(monkeypatch, client): - heartbeat = HeartbeatData(status=AgentStatus.READY, timestamp="2026-04-09T00:00:00Z", version="1.2.3") + heartbeat = HeartbeatData( + status=AgentStatus.READY, timestamp="2026-04-09T00:00:00Z", version="1.2.3" + ) - monkeypatch.setattr(client, "_async_request", AsyncMock(return_value=DummySyncResponse({"ok": True}))) - monkeypatch.setattr(client_mod.requests, "post", lambda *args, **kwargs: DummySyncResponse({"ok": True})) + monkeypatch.setattr( + client, + "_async_request", + AsyncMock(return_value=DummySyncResponse({"ok": True})), + ) + monkeypatch.setattr( + client_mod.requests, + "post", + lambda *args, **kwargs: DummySyncResponse({"ok": True}), + ) assert await client.send_enhanced_heartbeat("node-1", heartbeat) is True assert client.send_enhanced_heartbeat_sync("node-1", heartbeat) is True assert await client.notify_graceful_shutdown("node-1") is True assert client.notify_graceful_shutdown_sync("node-1") is True - monkeypatch.setattr(client, "_async_request", AsyncMock(side_effect=RuntimeError("boom"))) + monkeypatch.setattr( + client, "_async_request", AsyncMock(side_effect=RuntimeError("boom")) + ) monkeypatch.setattr( client_mod.requests, "post", @@ -654,8 +738,12 @@ async def test_heartbeat_and_shutdown_helpers(monkeypatch, client): @pytest.mark.asyncio async def test_async_execution_manager_wrappers(monkeypatch, client): manager = SimpleNamespace( - get_execution_status=AsyncMock(side_effect=[{"status": "running"}, RuntimeError("boom")]), - wait_for_result=AsyncMock(side_effect=[{"value": 1}, TimeoutError("late"), RuntimeError("broken")]), + get_execution_status=AsyncMock( + side_effect=[{"status": "running"}, RuntimeError("boom")] + ), + wait_for_result=AsyncMock( + side_effect=[{"value": 1}, TimeoutError("late"), RuntimeError("broken")] + ), cancel_execution=AsyncMock(side_effect=[True, False, RuntimeError("nope")]), list_executions=AsyncMock(return_value=[{"id": "exec-1"}]), get_metrics=lambda: {"active": 1}, @@ -667,14 +755,18 @@ async def test_async_execution_manager_wrappers(monkeypatch, client): client.async_config.enable_batch_polling = True client.async_config.batch_size = 2 - monkeypatch.setattr(client, "_get_async_execution_manager", AsyncMock(return_value=manager)) + monkeypatch.setattr( + client, "_get_async_execution_manager", AsyncMock(return_value=manager) + ) assert await client.poll_execution_status("exec-1") == {"status": "running"} with pytest.raises(AgentFieldClientError, match="Failed to poll execution status"): await client.poll_execution_status("exec-2") - manager.get_execution_status = AsyncMock(side_effect=[{"status": "done"}, None, None]) + manager.get_execution_status = AsyncMock( + side_effect=[{"status": "done"}, None, None] + ) batched = await client.batch_check_statuses(["a", "b", "c"]) assert batched == {"a": {"status": "done"}, "b": None, "c": None} @@ -682,7 +774,9 @@ async def test_async_execution_manager_wrappers(monkeypatch, client): await client.batch_check_statuses([]) client.async_config.enable_batch_polling = False - manager.get_execution_status = AsyncMock(side_effect=[{"status": "x"}, {"status": "y"}, {"status": "z"}]) + manager.get_execution_status = AsyncMock( + side_effect=[{"status": "x"}, {"status": "y"}, {"status": "z"}] + ) assert await client.batch_check_statuses(["a", "b", "c"]) == { "a": {"status": "x"}, "b": {"status": "y"}, @@ -694,7 +788,9 @@ async def test_async_execution_manager_wrappers(monkeypatch, client): with pytest.raises(ExecutionTimeoutError, match="exceeded timeout"): await client.wait_for_execution_result("exec-4") - with pytest.raises(AgentFieldClientError, match="Failed to wait for execution result"): + with pytest.raises( + AgentFieldClientError, match="Failed to wait for execution result" + ): await client.wait_for_execution_result("exec-5") assert await client.cancel_async_execution("exec-6", "done") is True @@ -716,8 +812,12 @@ async def test_async_execution_manager_wrappers(monkeypatch, client): async def test_approval_helpers_and_wait_for_approval(client, monkeypatch): client.caller_agent_id = "caller-1" router = respx.MockRouter(assert_all_called=True, assert_all_mocked=True) - async with httpx.AsyncClient(transport=httpx.MockTransport(router.async_handler)) as async_client: - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=async_client)) + async with httpx.AsyncClient( + transport=httpx.MockTransport(router.async_handler) + ) as async_client: + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=async_client) + ) request_route = router.post( "http://example.test/api/v1/agents/caller-1/executions/exec-1/request-approval" @@ -781,7 +881,9 @@ async def fake_get_status(execution_id): monkeypatch.setattr(client_mod.time, "time", lambda: next(times)) monkeypatch.setattr(client, "get_approval_status", fake_get_status) - waited = await client.wait_for_approval("exec-2", poll_interval=0.1, max_interval=0.2, timeout=5.0) + waited = await client.wait_for_approval( + "exec-2", poll_interval=0.1, max_interval=0.2, timeout=5.0 + ) assert waited.status == "approved" times_timeout = iter([0.0, 10.0]) @@ -795,9 +897,13 @@ async def fake_get_status(execution_id): async def test_approval_helpers_surface_http_errors_and_log_post_is_best_effort(client): client.caller_agent_id = "caller-1" router = respx.MockRouter(assert_all_called=True, assert_all_mocked=True) - async with httpx.AsyncClient(transport=httpx.MockTransport(router.async_handler)) as async_client: + async with httpx.AsyncClient( + transport=httpx.MockTransport(router.async_handler) + ) as async_client: monkeypatch = pytest.MonkeyPatch() - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=async_client)) + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=async_client) + ) try: router.post( "http://example.test/api/v1/agents/caller-1/executions/exec-4/request-approval" @@ -805,14 +911,16 @@ async def test_approval_helpers_surface_http_errors_and_log_post_is_best_effort( router.get( "http://example.test/api/v1/agents/caller-1/executions/exec-4/approval-status" ).mock(return_value=Response(502, text="bad gateway")) - log_route = router.post("http://example.test/api/v1/executions/exec-4/logs").mock( - side_effect=[Response(200, json={"ok": True}), RuntimeError("boom")] - ) + log_route = router.post( + "http://example.test/api/v1/executions/exec-4/logs" + ).mock(side_effect=[Response(200, json={"ok": True}), RuntimeError("boom")]) with pytest.raises(AgentFieldClientError, match="Approval request failed"): await client.request_approval("exec-4", "apr-4") - with pytest.raises(AgentFieldClientError, match="Approval status request failed"): + with pytest.raises( + AgentFieldClientError, match="Approval status request failed" + ): await client.get_approval_status("exec-4") await client.post_execution_logs("exec-4", {"message": "one"}) @@ -839,8 +947,12 @@ async def test_notify_awaiter_status_posts_waiting(client, monkeypatch): the body the control-plane handler expects.""" client.caller_agent_id = "middle-agent" router = respx.MockRouter(assert_all_called=True, assert_all_mocked=True) - async with httpx.AsyncClient(transport=httpx.MockTransport(router.async_handler)) as async_client: - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=async_client)) + async with httpx.AsyncClient( + transport=httpx.MockTransport(router.async_handler) + ) as async_client: + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=async_client) + ) route = router.post( "http://example.test/api/v1/agents/middle-agent/executions/middle-exec-1/awaiter-status" @@ -864,8 +976,12 @@ async def test_notify_awaiter_status_posts_running(client, monkeypatch): """Symmetric: status='running' completes the cascade pair.""" client.caller_agent_id = "middle-agent" router = respx.MockRouter(assert_all_called=True, assert_all_mocked=True) - async with httpx.AsyncClient(transport=httpx.MockTransport(router.async_handler)) as async_client: - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=async_client)) + async with httpx.AsyncClient( + transport=httpx.MockTransport(router.async_handler) + ) as async_client: + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=async_client) + ) route = router.post( "http://example.test/api/v1/agents/middle-agent/executions/middle-exec-1/awaiter-status" @@ -884,7 +1000,9 @@ async def test_notify_awaiter_status_rejects_bad_status(client): """The client validates ``status`` before going to the network — a bad value is a programmer error in the SDK call site, not a CP-side issue, so it surfaces as an immediate AgentFieldClientError.""" - with pytest.raises(AgentFieldClientError, match="status must be 'waiting' or 'running'"): + with pytest.raises( + AgentFieldClientError, match="status must be 'waiting' or 'running'" + ): await client.notify_awaiter_status( execution_id="exec-1", status="succeeded", # invalid for this endpoint @@ -898,8 +1016,12 @@ async def test_notify_awaiter_status_surfaces_http_errors(client, monkeypatch): so the error path isn't silently dropped at this layer.""" client.caller_agent_id = "middle-agent" router = respx.MockRouter(assert_all_called=True, assert_all_mocked=True) - async with httpx.AsyncClient(transport=httpx.MockTransport(router.async_handler)) as async_client: - monkeypatch.setattr(client, "get_async_http_client", AsyncMock(return_value=async_client)) + async with httpx.AsyncClient( + transport=httpx.MockTransport(router.async_handler) + ) as async_client: + monkeypatch.setattr( + client, "get_async_http_client", AsyncMock(return_value=async_client) + ) router.post( "http://example.test/api/v1/agents/middle-agent/executions/exec-bad/awaiter-status" diff --git a/sdk/python/tests/test_client_unit.py b/sdk/python/tests/test_client_unit.py index b38851baa..317f96526 100644 --- a/sdk/python/tests/test_client_unit.py +++ b/sdk/python/tests/test_client_unit.py @@ -122,7 +122,9 @@ def test_maybe_update_event_stream_headers_without_manager(source_headers, expec def test_restart_execution_posts_payload_and_headers(): client = AgentFieldClient(base_url="http://example.com", api_key="key-1") client._async_request = AsyncMock( - return_value=DummyResponse(202, {"execution_id": "exec-new", "run_id": "run-new"}) + return_value=DummyResponse( + 202, {"execution_id": "exec-new", "run_id": "run-new"} + ) ) result = asyncio.run( diff --git a/sdk/python/tests/test_connection_manager.py b/sdk/python/tests/test_connection_manager.py index 13dec7498..ca3b9ab7f 100644 --- a/sdk/python/tests/test_connection_manager.py +++ b/sdk/python/tests/test_connection_manager.py @@ -316,7 +316,10 @@ async def test_attempt_connection_pending_approval_blocks(self, mock_agent): """Test that pending_approval status blocks until approved (D1 fix).""" # Registration returns pending_approval mock_agent.client.register_agent_with_status = AsyncMock( - return_value=(True, {"status": "pending_approval", "pending_tags": ["sensitive"]}) + return_value=( + True, + {"status": "pending_approval", "pending_tags": ["sensitive"]}, + ) ) mock_agent.agent_tags = ["sensitive"] @@ -332,7 +335,9 @@ async def test_attempt_connection_pending_approval_blocks(self, mock_agent): mock_agent.agentfield_handler._wait_for_approval.assert_awaited_once() @pytest.mark.asyncio - async def test_attempt_connection_no_pending_approval_does_not_block(self, mock_agent): + async def test_attempt_connection_no_pending_approval_does_not_block( + self, mock_agent + ): """Test that non-pending registration does not call _wait_for_approval.""" mock_agent.client.register_agent_with_status = AsyncMock( return_value=(True, {"status": "ready"}) diff --git a/sdk/python/tests/test_connection_manager_invariants.py b/sdk/python/tests/test_connection_manager_invariants.py index f197e5ed9..aef5633d0 100644 --- a/sdk/python/tests/test_connection_manager_invariants.py +++ b/sdk/python/tests/test_connection_manager_invariants.py @@ -4,6 +4,7 @@ These tests verify the state machine properties of the ConnectionManager that must always hold regardless of implementation changes. """ + from __future__ import annotations import asyncio @@ -16,6 +17,7 @@ # Helpers # --------------------------------------------------------------------------- + def _make_mock_agent(agentfield_connected: bool = False): """Build a mock agent suitable for ConnectionManager construction.""" agent = MagicMock() @@ -51,7 +53,7 @@ def _make_connection_manager(agent=None): agent = _make_mock_agent() config = ConnectionConfig( - retry_interval=999.0, # long intervals so loops don't fire + retry_interval=999.0, # long intervals so loops don't fire health_check_interval=999.0, connection_timeout=1.0, ) @@ -62,6 +64,7 @@ def _make_connection_manager(agent=None): # 1. Valid state transitions # --------------------------------------------------------------------------- + class TestStateMachineValidTransitions: """Valid ConnectionState transitions must succeed without raising.""" @@ -82,7 +85,6 @@ async def test_invariant_disconnected_to_connecting_on_attempt(self): agent.client = MagicMock() agent.client.register_agent_with_status = AsyncMock(return_value=(False, None)) - original_attempt = manager._attempt_connection async def _tracking_attempt(): @@ -90,7 +92,9 @@ async def _tracking_attempt(): result = await original_attempt() return result - with patch.object(manager, "_attempt_connection", side_effect=_tracking_attempt): + with patch.object( + manager, "_attempt_connection", side_effect=_tracking_attempt + ): await manager._attempt_connection() # After a failed attempt, state should be DISCONNECTED @@ -150,6 +154,7 @@ async def test_invariant_connected_to_disconnected_on_failure(self): # 2. Invalid transitions must not occur directly # --------------------------------------------------------------------------- + class TestStateMachineInvalidTransitions: """DISCONNECTED → CONNECTED must not be a direct transition (must go through CONNECTING).""" @@ -182,7 +187,9 @@ async def test_invariant_no_direct_disconnected_to_connected_transition(self): ) @pytest.mark.asyncio - async def test_invariant_connected_to_connecting_is_not_possible_via_public_api(self): + async def test_invariant_connected_to_connecting_is_not_possible_via_public_api( + self, + ): """ force_reconnect() when already CONNECTED must return True immediately without changing state to CONNECTING (no spurious downgrade). @@ -208,6 +215,7 @@ async def test_invariant_connected_to_connecting_is_not_possible_via_public_api( # 3. Shutdown idempotency # --------------------------------------------------------------------------- + class TestShutdownIdempotency: """Calling stop() twice must not raise or corrupt state.""" @@ -255,15 +263,16 @@ async def _long_task(): f"INVARIANT VIOLATION: stop() with running task raised {type(exc).__name__}: {exc}" ) - assert manager._reconnection_task.cancelled() or manager._reconnection_task.done(), ( - "INVARIANT VIOLATION: stop() did not cancel the reconnection task." - ) + assert ( + manager._reconnection_task.cancelled() or manager._reconnection_task.done() + ), "INVARIANT VIOLATION: stop() did not cancel the reconnection task." # --------------------------------------------------------------------------- # 4. Reconnect after disconnect # --------------------------------------------------------------------------- + class TestReconnectAfterDisconnect: """After disconnect, the manager must be able to reconnect successfully.""" @@ -287,7 +296,9 @@ async def _successful_attempt(): manager.state = ConnectionState.CONNECTED return True - with patch.object(manager, "_attempt_connection", side_effect=_successful_attempt): + with patch.object( + manager, "_attempt_connection", side_effect=_successful_attempt + ): result = await manager.force_reconnect() assert result is True, ( diff --git a/sdk/python/tests/test_crypto.py b/sdk/python/tests/test_crypto.py index 4a80ad43e..579ac9f48 100644 --- a/sdk/python/tests/test_crypto.py +++ b/sdk/python/tests/test_crypto.py @@ -74,7 +74,9 @@ def test_encrypt_for_did_unwraps_resolve_response(): """The control-plane resolve endpoint wraps the document under did_document.""" priv, pub = crypto.generate_x25519_keypair() resolve_response = {"did": "did:key:zAgg", "did_document": _did_document(pub)} - token = crypto.encrypt_for_did("did:key:zAgg", "wrapped", resolver=lambda _d: resolve_response) + token = crypto.encrypt_for_did( + "did:key:zAgg", "wrapped", resolver=lambda _d: resolve_response + ) assert crypto.decrypt(token, priv) == b"wrapped" @@ -82,7 +84,9 @@ def test_encrypt_for_did_accepts_flat_key_agreement(): """The CP did:key resolve response returns a flat `key_agreement` JWK.""" priv, pub = crypto.generate_x25519_keypair() resolve_response = {"did": "did:key:zAgg", "key_agreement": pub} - token = crypto.encrypt_for_did("did:key:zAgg", "flat", resolver=lambda _d: resolve_response) + token = crypto.encrypt_for_did( + "did:key:zAgg", "flat", resolver=lambda _d: resolve_response + ) assert crypto.decrypt(token, priv) == b"flat" @@ -143,7 +147,9 @@ def test_load_private_key_missing_env(monkeypatch): def test_sign_verify_roundtrip_dict(): priv, pub = crypto.generate_ed25519_keypair() - payload = {"scope": {"tier": "project", "workspace_id": "ws_1", "project_id": "p_2"}} + payload = { + "scope": {"tier": "project", "workspace_id": "ws_1", "project_id": "p_2"} + } assert json.loads(crypto.verify(crypto.sign(payload, priv), pub)) == payload diff --git a/sdk/python/tests/test_decorator_code_origin.py b/sdk/python/tests/test_decorator_code_origin.py index b0522e4b9..ce7fc6d07 100644 --- a/sdk/python/tests/test_decorator_code_origin.py +++ b/sdk/python/tests/test_decorator_code_origin.py @@ -44,9 +44,7 @@ def test_agent_reasoner_trigger_metadata_stamps_code_origin(self, monkeypatch): async def handle_agent_payment(payload: dict) -> dict: return payload - metadata = next( - r for r in app.reasoners if r["id"] == "handle_agent_payment" - ) + metadata = next(r for r in app.reasoners if r["id"] == "handle_agent_payment") code_origin = metadata["triggers"][0]["code_origin"] assert code_origin is not None @@ -63,9 +61,7 @@ def test_agent_reasoner_trigger_metadata_unwraps_inner_reasoner(self, monkeypatc async def wrapped_agent_payment(payload: dict) -> dict: return payload - metadata = next( - r for r in app.reasoners if r["id"] == "wrapped_agent_payment" - ) + metadata = next(r for r in app.reasoners if r["id"] == "wrapped_agent_payment") code_origin = metadata["triggers"][0]["code_origin"] assert code_origin is not None diff --git a/sdk/python/tests/test_did_auth.py b/sdk/python/tests/test_did_auth.py index 6c80e7d65..5dd9204b2 100644 --- a/sdk/python/tests/test_did_auth.py +++ b/sdk/python/tests/test_did_auth.py @@ -25,6 +25,7 @@ # Helpers: generate a real Ed25519 key pair in JWK format # --------------------------------------------------------------------------- + def _generate_ed25519_jwk(): """Generate a fresh Ed25519 key pair and return (private_key_jwk_str, public_key_obj, private_key_obj).""" private_key = Ed25519PrivateKey.generate() @@ -70,6 +71,7 @@ def did_string(): # Tests for _load_ed25519_private_key # =========================================================================== + class TestLoadEd25519PrivateKey: """Tests for _load_ed25519_private_key.""" @@ -160,6 +162,7 @@ def test_non_json_string_raises_valueerror(self): # Tests for sign_request # =========================================================================== + class TestSignRequest: """Tests for the sign_request function.""" @@ -208,14 +211,18 @@ def test_signature_verifies_with_public_key(self, ed25519_jwk, did_string): # verify raises InvalidSignature on failure public_key.verify(sig_bytes, payload) - def test_different_bodies_produce_different_signatures(self, ed25519_jwk, did_string): + def test_different_bodies_produce_different_signatures( + self, ed25519_jwk, did_string + ): jwk_str, _, _ = ed25519_jwk sig1, _, _, _ = sign_request(b"body_a", jwk_str, did_string) sig2, _, _, _ = sign_request(b"body_b", jwk_str, did_string) # Signatures should differ (different body hash) assert sig1 != sig2 - def test_same_body_produces_different_signatures_via_nonce(self, ed25519_jwk, did_string): + def test_same_body_produces_different_signatures_via_nonce( + self, ed25519_jwk, did_string + ): """Two calls with the same body should produce different signatures due to nonce.""" jwk_str, _, _ = ed25519_jwk sig1, _, nonce1, _ = sign_request(b"same body", jwk_str, did_string) @@ -233,6 +240,7 @@ def test_invalid_key_raises(self, did_string): # Tests for create_did_auth_headers # =========================================================================== + class TestCreateDIDAuthHeaders: """Tests for the create_did_auth_headers convenience function.""" @@ -266,6 +274,7 @@ def test_nonce_header_is_hex(self, ed25519_jwk, did_string): # Tests for DIDAuthenticator # =========================================================================== + class TestDIDAuthenticator: """Tests for the DIDAuthenticator class.""" @@ -388,6 +397,7 @@ def test_get_auth_info_configured(self, ed25519_jwk, did_string): # Edge cases # =========================================================================== + class TestEdgeCases: """Edge-case tests for body signing.""" @@ -411,7 +421,9 @@ def test_large_body(self, ed25519_jwk, did_string): def test_non_ascii_body(self, ed25519_jwk, did_string): jwk_str, public_key, _ = ed25519_jwk - body = "Unicode payload: \u00e9\u00e8\u00ea \u4e16\u754c \U0001f680".encode("utf-8") + body = "Unicode payload: \u00e9\u00e8\u00ea \u4e16\u754c \U0001f680".encode( + "utf-8" + ) sig_b64, ts, nonce, _ = sign_request(body, jwk_str, did_string) body_hash = hashlib.sha256(body).hexdigest() payload = f"{ts}:{nonce}:{body_hash}".encode("utf-8") diff --git a/sdk/python/tests/test_did_auth_invariants.py b/sdk/python/tests/test_did_auth_invariants.py index dc95f0f69..124cebede 100644 --- a/sdk/python/tests/test_did_auth_invariants.py +++ b/sdk/python/tests/test_did_auth_invariants.py @@ -5,6 +5,7 @@ always hold regardless of implementation changes. They are designed to catch AI code regressions that break cryptographic contracts. """ + from __future__ import annotations import base64 @@ -20,6 +21,7 @@ # Fixtures # --------------------------------------------------------------------------- + @pytest.fixture def ed25519_key_pair(): """Generate a fresh Ed25519 key pair in JWK format for testing.""" @@ -44,9 +46,11 @@ def ed25519_key_pair(): # Helper to call sign_request # --------------------------------------------------------------------------- + def _sign(body: bytes, jwk: str, did: str) -> Tuple[str, str, str, str]: """Call sign_request and return (signature, timestamp, nonce, did).""" from agentfield.did_auth import sign_request + return sign_request(body, jwk, did) @@ -54,6 +58,7 @@ def _sign(body: bytes, jwk: str, did: str) -> Tuple[str, str, str, str]: # 1. Nonce uniqueness # --------------------------------------------------------------------------- + class TestNonceUniqueness: """Nonces must be globally unique across many rapid calls.""" @@ -91,6 +96,7 @@ def test_invariant_nonce_is_unique_for_identical_body(self, ed25519_key_pair): # 2. Timestamp freshness # --------------------------------------------------------------------------- + class TestTimestampFreshness: """Every signed request's timestamp must be within 5 seconds of now.""" @@ -128,11 +134,14 @@ def test_invariant_timestamp_is_a_numeric_string(self, ed25519_key_pair): # 3. Signature determinism # --------------------------------------------------------------------------- + class TestSignatureDeterminism: """Same inputs (excluding nonce/timestamp) signed with same key and EXPLICIT nonce/timestamp must produce identical signatures — Ed25519 is deterministic.""" - def test_invariant_signature_is_deterministic_for_same_payload(self, ed25519_key_pair): + def test_invariant_signature_is_deterministic_for_same_payload( + self, ed25519_key_pair + ): """ Rebuild the exact payload that sign_request would sign, manually, and verify that two independent sign calls produce the same signature @@ -142,7 +151,9 @@ def test_invariant_signature_is_deterministic_for_same_payload(self, ed25519_key same signature out (Ed25519 determinism property). """ try: - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey # noqa: F401 + from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + ) # noqa: F401 except ImportError: pytest.skip("cryptography library not available") @@ -157,6 +168,7 @@ def test_invariant_signature_is_deterministic_for_same_payload(self, ed25519_key # Load private key and sign twice from agentfield.did_auth import _load_ed25519_private_key + private_key = _load_ed25519_private_key(jwk) sig1 = base64.b64encode(private_key.sign(payload)).decode("ascii") @@ -172,11 +184,13 @@ def test_invariant_signature_is_deterministic_for_same_payload(self, ed25519_key # 4. Signature sensitivity # --------------------------------------------------------------------------- + class TestSignatureSensitivity: """Changing ANY part of the signed input must produce a different signature.""" - def _sign_with_patched_inputs(self, jwk: str, did: str, body: bytes, - timestamp: str, nonce: str) -> str: + def _sign_with_patched_inputs( + self, jwk: str, did: str, body: bytes, timestamp: str, nonce: str + ) -> str: """Sign with fully controlled inputs by patching os.urandom and time.time.""" from agentfield.did_auth import _load_ed25519_private_key @@ -186,7 +200,9 @@ def _sign_with_patched_inputs(self, jwk: str, did: str, body: bytes, sig = private_key.sign(payload) return base64.b64encode(sig).decode("ascii") - def test_invariant_different_body_produces_different_signature(self, ed25519_key_pair): + def test_invariant_different_body_produces_different_signature( + self, ed25519_key_pair + ): jwk, did = ed25519_key_pair ts = "1700000000" nonce = "fixed_nonce_0123456789ab" @@ -198,7 +214,9 @@ def test_invariant_different_body_produces_different_signature(self, ed25519_key "INVARIANT VIOLATION: Different bodies produced the same signature." ) - def test_invariant_different_timestamp_produces_different_signature(self, ed25519_key_pair): + def test_invariant_different_timestamp_produces_different_signature( + self, ed25519_key_pair + ): jwk, did = ed25519_key_pair body = b'{"fixed": "body"}' nonce = "fixed_nonce_0123456789ab" @@ -210,7 +228,9 @@ def test_invariant_different_timestamp_produces_different_signature(self, ed2551 "INVARIANT VIOLATION: Different timestamps produced the same signature." ) - def test_invariant_different_nonce_produces_different_signature(self, ed25519_key_pair): + def test_invariant_different_nonce_produces_different_signature( + self, ed25519_key_pair + ): jwk, did = ed25519_key_pair body = b'{"fixed": "body"}' ts = "1700000000" @@ -227,6 +247,7 @@ def test_invariant_different_nonce_produces_different_signature(self, ed25519_ke # 5. Header completeness # --------------------------------------------------------------------------- + class TestHeaderCompleteness: """Every signed request must contain ALL required DID authentication headers.""" @@ -284,7 +305,9 @@ def test_invariant_authenticator_sign_headers_returns_all_required_when_configur jwk, did = ed25519_key_pair auth = DIDAuthenticator(did=did, private_key_jwk=jwk) - assert auth.is_configured, "DIDAuthenticator must be configured after __init__ with valid key" + assert auth.is_configured, ( + "DIDAuthenticator must be configured after __init__ with valid key" + ) headers = auth.sign_headers(b'{"payload": "data"}') diff --git a/sdk/python/tests/test_exceptions.py b/sdk/python/tests/test_exceptions.py index f3d034b95..9e6d598fb 100644 --- a/sdk/python/tests/test_exceptions.py +++ b/sdk/python/tests/test_exceptions.py @@ -189,16 +189,12 @@ def test_submit_execution_network_error_raises_client_error(self): ) client = AgentFieldClient(base_url="http://localhost:8080") with pytest.raises(AgentFieldClientError, match="Failed to submit execution"): - client._submit_execution_sync( - "agent.skill", {"key": "value"}, {} - ) + client._submit_execution_sync("agent.skill", {"key": "value"}, {}) @responses_lib.activate def test_parse_submission_missing_ids_raises_client_error(self): client = AgentFieldClient(base_url="http://localhost:8080") - with pytest.raises( - AgentFieldClientError, match="missing identifiers" - ): + with pytest.raises(AgentFieldClientError, match="missing identifiers"): client._parse_submission( {"status": "pending"}, # no execution_id or run_id {"X-Run-ID": ""}, @@ -316,57 +312,43 @@ def memory_client(self): @pytest.mark.asyncio async def test_set_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to set memory key"): await memory_client.set("key", "value") @pytest.mark.asyncio async def test_get_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to get memory key"): await memory_client.get("key") @pytest.mark.asyncio async def test_delete_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to delete memory key"): await memory_client.delete("key") @pytest.mark.asyncio async def test_list_keys_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to list keys"): await memory_client.list_keys("global") @pytest.mark.asyncio async def test_set_vector_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to set vector key"): await memory_client.set_vector("key", [0.1, 0.2]) @pytest.mark.asyncio async def test_delete_vector_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to delete vector key"): await memory_client.delete_vector("key") @pytest.mark.asyncio async def test_similarity_search_wraps_transport_error(self, memory_client): - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) with pytest.raises(MemoryAccessError, match="Failed to perform similarity"): await memory_client.similarity_search([0.1, 0.2]) @@ -383,9 +365,7 @@ async def test_get_returns_default_on_404(self, memory_client): @pytest.mark.asyncio async def test_exists_returns_false_on_error(self, memory_client): """exists() intentionally suppresses errors and returns False.""" - memory_client._async_request = AsyncMock( - side_effect=ConnectionError("refused") - ) + memory_client._async_request = AsyncMock(side_effect=ConnectionError("refused")) result = await memory_client.exists("key") assert result is False diff --git a/sdk/python/tests/test_execution_context_parent_vc.py b/sdk/python/tests/test_execution_context_parent_vc.py index f46a8415a..0c98b3150 100644 --- a/sdk/python/tests/test_execution_context_parent_vc.py +++ b/sdk/python/tests/test_execution_context_parent_vc.py @@ -59,6 +59,7 @@ def test_to_headers_omits_parent_vc_id_when_none(): @pytest.mark.unit def test_from_request_reads_parent_vc_id(): """Verify that from_request() reads X-Parent-VC-ID header and sets parent_vc_id field.""" + # Mock FastAPI request headers class MockHeaders: def __init__(self, data): @@ -71,12 +72,14 @@ class MockRequest: def __init__(self, headers_dict): self.headers = MockHeaders(headers_dict) - request = MockRequest({ - "X-Workflow-ID": "wf-1", - "X-Execution-ID": "exec-1", - "X-Parent-VC-ID": "vc_trigger_webhook_456", - "X-Run-ID": "run-1", - }) + request = MockRequest( + { + "X-Workflow-ID": "wf-1", + "X-Execution-ID": "exec-1", + "X-Parent-VC-ID": "vc_trigger_webhook_456", + "X-Run-ID": "run-1", + } + ) ctx = ExecutionContext.from_request(request, agent_node_id="node-1") @@ -86,6 +89,7 @@ def __init__(self, headers_dict): @pytest.mark.unit def test_from_request_handles_missing_parent_vc_id(): """Verify that from_request() handles missing X-Parent-VC-ID gracefully.""" + class MockHeaders: def __init__(self, data): self._data = data @@ -97,11 +101,13 @@ class MockRequest: def __init__(self, headers_dict): self.headers = MockHeaders(headers_dict) - request = MockRequest({ - "X-Workflow-ID": "wf-1", - "X-Execution-ID": "exec-1", - "X-Run-ID": "run-1", - }) + request = MockRequest( + { + "X-Workflow-ID": "wf-1", + "X-Execution-ID": "exec-1", + "X-Run-ID": "run-1", + } + ) ctx = ExecutionContext.from_request(request, agent_node_id="node-1") diff --git a/sdk/python/tests/test_execution_logger.py b/sdk/python/tests/test_execution_logger.py index 453402059..ea4c5ff9b 100644 --- a/sdk/python/tests/test_execution_logger.py +++ b/sdk/python/tests/test_execution_logger.py @@ -155,10 +155,11 @@ def capture(*args, **kwargs): assert captured[1]["attributes"]["result"] == {"ok": True} assert captured[2]["attributes"]["error"] == "boom" + @pytest.fixture def base_logger(monkeypatch): """ - Initializes an AgentFieldLogger with environment overrides to validate + Initializes an AgentFieldLogger with environment overrides to validate core observability and data-handling logic. """ monkeypatch.setenv("AGENTFIELD_LOG_LEVEL", "DEBUG") @@ -166,41 +167,48 @@ def base_logger(monkeypatch): monkeypatch.setenv("AGENTFIELD_LOG_TRUNCATE", "50") monkeypatch.setenv("AGENTFIELD_LOG_TRACKING", "true") monkeypatch.setenv("AGENTFIELD_LOG_FIRE", "true") - + logger = AgentFieldLogger(name="telemetry") logger.logger.propagate = True return logger + @pytest.mark.unit def test_heartbeat_event(base_logger, caplog): base_logger.heartbeat("Agent pulsing", status="nominal") assert "Agent pulsing" in caplog.text + @pytest.mark.unit def test_logger_track_output(base_logger, caplog): base_logger.track("token_usage", count=500) assert "token_usage" in caplog.text + @pytest.mark.unit def test_logger_fire_output(base_logger, caplog): base_logger.fire("node_transition", target="reasoning_node") assert "node_transition" in caplog.text + @pytest.mark.unit def test_logger_debug_output(base_logger, caplog): base_logger.debug("Debugging logic") assert "Debugging" in caplog.text + @pytest.mark.unit def test_logger_security_output(base_logger, caplog): base_logger.security("Sanitized keys", level="HIGH") assert "Sanitized" in caplog.text + @pytest.mark.unit def test_logger_network_output(base_logger, caplog): base_logger.network("GET https://api.openai.com", method="GET") assert "api.openai.com" in caplog.text + @pytest.mark.unit def test_logger_severity_levels(base_logger, caplog): base_logger.warn("Warning msg") @@ -209,7 +217,8 @@ def test_logger_severity_levels(base_logger, caplog): out = caplog.text assert "Warning" in out assert "Error" in out - assert "Failure" in out + assert "Failure" in out + @pytest.mark.unit def test_logger_success_and_setup(base_logger, caplog): @@ -218,8 +227,10 @@ def test_logger_success_and_setup(base_logger, caplog): out = caplog.text assert "Operation" in out and "Environment" in out + # --- FORMATTING & PAYLOAD EDGE CASES --- + @pytest.mark.unit def test_truncate_message_at_limit(base_logger): """Verifies message longer than truncate_length -> ends with '...'""" @@ -228,12 +239,14 @@ def test_truncate_message_at_limit(base_logger): assert len(truncated) == 53 assert truncated.endswith("...") + @pytest.mark.unit def test_logger_short_message_handling(base_logger): """Verifies that messages under the truncate limit are untouched""" msg = "Short" assert base_logger._truncate_message(msg) == "Short" + @pytest.mark.unit def test_format_payload_hides_by_default(monkeypatch): """Verifies dict -> '[payload hidden]' when flag is false""" @@ -243,6 +256,7 @@ def test_format_payload_hides_by_default(monkeypatch): formatted = logger._format_payload(test_data) assert formatted == "[payload hidden - set AGENTFIELD_LOG_PAYLOADS=true to show]" + @pytest.mark.unit def test_format_payload_shows_when_enabled(base_logger): """Verifies AGENTFIELD_LOG_PAYLOADS=true -> JSON string""" @@ -251,9 +265,10 @@ def test_format_payload_shows_when_enabled(base_logger): decoded = json.loads(formatted) assert decoded["id"] == "123" + @pytest.mark.unit def test_format_payload_handles_non_serializable(base_logger): """Verifies fallback to str() for objects with no __dict__ or JSON support""" test_data = {1, 2, 3} formatted = base_logger._format_payload(test_data) - assert "{1, 2, 3}" in formatted \ No newline at end of file + assert "{1, 2, 3}" in formatted diff --git a/sdk/python/tests/test_execution_state_invariants.py b/sdk/python/tests/test_execution_state_invariants.py index 456f72fe5..683f2469c 100644 --- a/sdk/python/tests/test_execution_state_invariants.py +++ b/sdk/python/tests/test_execution_state_invariants.py @@ -5,18 +5,20 @@ of implementation changes, protecting against AI regressions that break the execution lifecycle contract. """ + from __future__ import annotations import time - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _make_state(execution_id: str = "exec-001", target: str = "node.reasoner", - status=None): + +def _make_state( + execution_id: str = "exec-001", target: str = "node.reasoner", status=None +): """Create a minimal ExecutionState with optional initial status.""" from agentfield.execution_state import ExecutionState @@ -34,17 +36,28 @@ def _make_state(execution_id: str = "exec-001", target: str = "node.reasoner", # 1. Status normalization idempotency # --------------------------------------------------------------------------- + class TestStatusNormalizationIdempotency: """normalize(normalize(x)) == normalize(x) for all canonical status values.""" def _normalize(self, status: str) -> str: from agentfield.status import normalize_status + return normalize_status(status) def test_invariant_normalize_is_idempotent_for_all_canonical_values(self): """normalize(normalize(x)) == normalize(x) for every canonical status.""" - canonical = ["pending", "queued", "waiting", "running", - "succeeded", "failed", "cancelled", "timeout", "unknown"] + canonical = [ + "pending", + "queued", + "waiting", + "running", + "succeeded", + "failed", + "cancelled", + "timeout", + "unknown", + ] for status in canonical: once = self._normalize(status) twice = self._normalize(once) @@ -56,9 +69,19 @@ def test_invariant_normalize_is_idempotent_for_all_canonical_values(self): def test_invariant_normalize_is_idempotent_for_aliases(self): """normalize(normalize(alias)) == normalize(alias) for all known aliases.""" aliases = [ - "success", "error", "completed", "done", "ok", - "canceled", "timed_out", "in_progress", "processing", - "RUNNING", "FAILED", "Success", "Error", + "success", + "error", + "completed", + "done", + "ok", + "canceled", + "timed_out", + "in_progress", + "processing", + "RUNNING", + "FAILED", + "Success", + "Error", ] for alias in aliases: once = self._normalize(alias) @@ -73,6 +96,7 @@ def test_invariant_normalize_is_idempotent_for_aliases(self): # 2. Terminal state detection consistency # --------------------------------------------------------------------------- + class TestTerminalStateConsistency: """is_terminal must agree with the TERMINAL_STATUSES set.""" @@ -132,6 +156,7 @@ def test_invariant_terminal_and_active_are_disjoint(self): # 3. Serialization roundtrip # --------------------------------------------------------------------------- + class TestSerializationRoundtrip: """ExecutionState.to_dict() must preserve all observable state.""" @@ -141,11 +166,24 @@ def test_invariant_to_dict_contains_all_required_keys(self): d = state.to_dict() required_keys = [ - "execution_id", "target", "status", "priority", - "created_at", "updated_at", - "result", "error_message", "error_details", - "workflow_id", "parent_execution_id", "session_id", "actor_id", - "timeout", "is_terminal", "is_active", "is_successful", "is_cancelled", + "execution_id", + "target", + "status", + "priority", + "created_at", + "updated_at", + "result", + "error_message", + "error_details", + "workflow_id", + "parent_execution_id", + "session_id", + "actor_id", + "timeout", + "is_terminal", + "is_active", + "is_successful", + "is_cancelled", "metrics", ] for key in required_keys: @@ -209,6 +247,7 @@ def test_invariant_to_dict_metrics_contains_poll_count(self): # 4. Metrics monotonicity # --------------------------------------------------------------------------- + class TestMetricsMonotonicity: """poll_count must only increase, never decrease.""" @@ -268,6 +307,7 @@ def test_invariant_poll_count_never_decreases_on_failure(self): # 5. Timestamp ordering # --------------------------------------------------------------------------- + class TestTimestampOrdering: """When both are set, started_at <= completed_at must always hold.""" diff --git a/sdk/python/tests/test_harness_ai_schema_repair.py b/sdk/python/tests/test_harness_ai_schema_repair.py index 0528a7b74..5e64ac306 100644 --- a/sdk/python/tests/test_harness_ai_schema_repair.py +++ b/sdk/python/tests/test_harness_ai_schema_repair.py @@ -167,9 +167,7 @@ async def _hang(*args, **kwargs): await asyncio.sleep(3600) # would hang for an hour # Patch the module-level timeout to something the test can wait for. - with patch( - "agentfield.harness._runner._SCHEMA_REPAIR_TIMEOUT_SECONDS", 0.1 - ): + with patch("agentfield.harness._runner._SCHEMA_REPAIR_TIMEOUT_SECONDS", 0.1): with patch("litellm.acompletion", side_effect=_hang): result = await _ai_schema_repair("some text", _Schema, options) diff --git a/sdk/python/tests/test_harness_factory.py b/sdk/python/tests/test_harness_factory.py index 8c3faccf7..84ad30657 100644 --- a/sdk/python/tests/test_harness_factory.py +++ b/sdk/python/tests/test_harness_factory.py @@ -173,6 +173,7 @@ def test_build_provider_claude_code_returns_claude_provider(): def test_harness_provider_protocol_is_runtime_checkable(): """HarnessProvider is a @runtime_checkable Protocol.""" + # An object without execute() should not satisfy the protocol class _NoExecute: pass diff --git a/sdk/python/tests/test_invariants.py b/sdk/python/tests/test_invariants.py index 7337cd305..f4a6759f4 100644 --- a/sdk/python/tests/test_invariants.py +++ b/sdk/python/tests/test_invariants.py @@ -12,6 +12,7 @@ 4. Independence — scopes, sessions, and agents that must not leak state 5. Schema stability — API shapes that clients depend on """ + from __future__ import annotations import json @@ -21,72 +22,115 @@ # Try importing hypothesis for property-based testing try: from hypothesis import given, settings, strategies as st + HAS_HYPOTHESIS = True except ImportError: HAS_HYPOTHESIS = False + # Provide no-op decorators so tests are skipped cleanly def given(*a, **kw): def decorator(fn): return pytest.mark.skip(reason="hypothesis not installed")(fn) + return decorator + def settings(*a, **kw): def decorator(fn): return fn + return decorator + class st: @staticmethod - def text(**kw): return None + def text(**kw): + return None + @staticmethod - def sampled_from(x): return None + def sampled_from(x): + return None + @staticmethod - def dictionaries(k, v): return None + def dictionaries(k, v): + return None + @staticmethod - def integers(**kw): return None + def integers(**kw): + return None + @staticmethod - def booleans(): return None + def booleans(): + return None + @staticmethod - def none(): return None + def none(): + return None + @staticmethod - def one_of(*a): return None + def one_of(*a): + return None # --------------------------------------------------------------------------- # 1. Status normalization invariants # --------------------------------------------------------------------------- + class TestStatusNormalizationInvariants: """The status normalization system maps many aliases to canonical forms. These invariants must hold even if AI adds new aliases or renames statuses.""" - CANONICAL = ["pending", "queued", "waiting", "running", - "succeeded", "failed", "cancelled", "timeout", "unknown"] + CANONICAL = [ + "pending", + "queued", + "waiting", + "running", + "succeeded", + "failed", + "cancelled", + "timeout", + "unknown", + ] TERMINAL = ["succeeded", "failed", "cancelled", "timeout"] ACTIVE = ["pending", "queued", "waiting", "running"] def _normalize(self, status: str) -> str: """Import and call the real normalize function.""" from agentfield.status import normalize_status + return normalize_status(status) def test_canonical_statuses_are_fixed_points(self): """normalize(canonical) == canonical — always.""" for status in self.CANONICAL: - assert self._normalize(status) == status, \ + assert self._normalize(status) == status, ( f"Canonical status '{status}' is not a fixed point of normalize()" + ) def test_normalize_is_idempotent(self): """normalize(normalize(x)) == normalize(x) for all x.""" test_values = self.CANONICAL + [ - "success", "error", "completed", "done", "ok", - "canceled", "timed_out", "in_progress", "processing", - "RUNNING", " failed ", "Unknown", "", "gibberish" + "success", + "error", + "completed", + "done", + "ok", + "canceled", + "timed_out", + "in_progress", + "processing", + "RUNNING", + " failed ", + "Unknown", + "", + "gibberish", ] for val in test_values: once = self._normalize(val) twice = self._normalize(once) - assert once == twice, \ - f"normalize is not idempotent: normalize('{val}')='{once}', " \ + assert once == twice, ( + f"normalize is not idempotent: normalize('{val}')='{once}', " f"normalize('{once}')='{twice}'" + ) def test_terminal_and_active_are_disjoint(self): """No status can be both terminal and active.""" @@ -102,16 +146,18 @@ def test_terminal_and_active_cover_all_actionable_canonical(self): for status in self.CANONICAL: if status == "unknown": continue # sentinel, not an actionable state - assert status in covered, \ + assert status in covered, ( f"Canonical status '{status}' is neither terminal nor active" + ) def test_normalize_never_returns_empty(self): """Normalize must always return a non-empty string.""" for val in ["", " ", None, "garbage", "💀", "a" * 1000]: try: result = self._normalize(str(val) if val is not None else "") - assert result and len(result) > 0, \ + assert result and len(result) > 0, ( f"normalize('{val}') returned empty: '{result}'" + ) except (TypeError, AttributeError): pass # None might raise, that's fine @@ -128,6 +174,7 @@ def test_normalize_never_crashes_on_arbitrary_input(self, status): # 2. Memory scope independence # --------------------------------------------------------------------------- + class TestMemoryScopeInvariant: """Memory scopes must be fully independent. Setting a key in one scope must never affect the same key in another scope.""" @@ -146,12 +193,14 @@ def test_scope_isolation_in_handler_routing(self): # 3. Execution context invariants # --------------------------------------------------------------------------- + class TestExecutionContextInvariants: """Execution context must be properly created and cleaned up.""" def test_context_cleanup_after_success(self): """After handle_serverless succeeds, _current_execution_context must be None.""" from agentfield.agent import Agent + agent = Agent(node_id="ctx-test") @agent.reasoner("noop") @@ -160,12 +209,14 @@ def noop(): result = agent.handle_serverless({"reasoner": "noop", "input": {}}) assert result["statusCode"] == 200 - assert agent._current_execution_context is None, \ + assert agent._current_execution_context is None, ( "INVARIANT: execution context must be cleaned up after success" + ) def test_context_cleanup_after_failure(self): """After handle_serverless fails, _current_execution_context must still be None.""" from agentfield.agent import Agent + agent = Agent(node_id="ctx-test-fail") @agent.reasoner("boom") @@ -174,12 +225,14 @@ def boom(): result = agent.handle_serverless({"reasoner": "boom", "input": {}}) assert result["statusCode"] == 500 - assert agent._current_execution_context is None, \ + assert agent._current_execution_context is None, ( "INVARIANT: execution context must be cleaned up even after failure" + ) def test_context_not_leaked_between_calls(self): """Two sequential serverless calls must not share execution context.""" from agentfield.agent import Agent + agent = Agent(node_id="ctx-leak-test") captured_contexts = [] @@ -189,26 +242,32 @@ def capture(): captured_contexts.append(agent._current_execution_context) return {} - agent.handle_serverless({ - "reasoner": "capture", - "input": {}, - "execution_context": {"execution_id": "exec-1"}, - }) - agent.handle_serverless({ - "reasoner": "capture", - "input": {}, - "execution_context": {"execution_id": "exec-2"}, - }) + agent.handle_serverless( + { + "reasoner": "capture", + "input": {}, + "execution_context": {"execution_id": "exec-1"}, + } + ) + agent.handle_serverless( + { + "reasoner": "capture", + "input": {}, + "execution_context": {"execution_id": "exec-2"}, + } + ) assert len(captured_contexts) == 2 - assert captured_contexts[0].execution_id != captured_contexts[1].execution_id, \ + assert captured_contexts[0].execution_id != captured_contexts[1].execution_id, ( "INVARIANT: sequential calls must have distinct execution contexts" + ) # --------------------------------------------------------------------------- # 4. Serialization roundtrip invariants # --------------------------------------------------------------------------- + class TestSerializationInvariants: """JSON serialization must be lossless for all SDK types.""" @@ -230,14 +289,16 @@ def test_memory_data_roundtrip_preserves_types(self): for val in test_values: serialized = json.dumps(val) deserialized = json.loads(serialized) - assert deserialized == val, \ + assert deserialized == val, ( f"Roundtrip failed for {type(val).__name__}: {val} → {deserialized}" + ) # --------------------------------------------------------------------------- # 5. Discovery response schema stability # --------------------------------------------------------------------------- + class TestDiscoverySchemaStability: """The discovery response schema is a contract between SDK and control plane. AI must not change field names or add/remove required fields.""" @@ -245,6 +306,7 @@ class TestDiscoverySchemaStability: def test_discovery_response_has_required_fields(self): """Verify discovery response contains all fields the control plane expects.""" from agentfield.agent import Agent + agent = Agent(node_id="schema-test") @agent.reasoner("greet") @@ -253,7 +315,11 @@ def greet(name: str = "world") -> dict: # Get discovery response discovery = agent.handle_serverless({"path": "/discover"}) - body = discovery if isinstance(discovery, dict) and "reasoners" in discovery else discovery.get("body", discovery) + body = ( + discovery + if isinstance(discovery, dict) and "reasoners" in discovery + else discovery.get("body", discovery) + ) # If this is a statusCode/body wrapper, unwrap if isinstance(body, dict) and "body" in body: @@ -262,14 +328,16 @@ def greet(name: str = "world") -> dict: # These fields MUST exist — they're the control plane contract required_top_level = {"node_id"} for field in required_top_level: - assert field in body, \ + assert field in body, ( f"SCHEMA VIOLATION: discovery response missing required field '{field}'" + ) # --------------------------------------------------------------------------- # 6. ProcessLogRing monotonicity # --------------------------------------------------------------------------- + class TestLogRingMonotonicity: """Log sequence numbers must be strictly monotonically increasing. AI might reset the counter or use non-atomic increments.""" @@ -289,8 +357,9 @@ def test_sequence_numbers_are_monotonic(self): seqs = [e.seq for e in entries] for i in range(1, len(seqs)): - assert seqs[i] > seqs[i - 1], \ - f"MONOTONICITY VIOLATION: seq[{i-1}]={seqs[i-1]} >= seq[{i}]={seqs[i]}" + assert seqs[i] > seqs[i - 1], ( + f"MONOTONICITY VIOLATION: seq[{i - 1}]={seqs[i - 1]} >= seq[{i}]={seqs[i]}" + ) def test_sequence_survives_eviction(self): """Even after buffer wraps and evicts old entries, new entries @@ -308,8 +377,9 @@ def test_sequence_survives_eviction(self): entries = list(ring.tail(1000)) if entries: current_max = max(e.seq for e in entries) - assert current_max >= max_seq_seen, \ + assert current_max >= max_seq_seen, ( f"Eviction caused seq regression: {current_max} < {max_seq_seen}" + ) max_seq_seen = current_max @@ -317,27 +387,30 @@ def test_sequence_survives_eviction(self): # 7. Cross-cutting: error responses are always structured # --------------------------------------------------------------------------- + class TestErrorResponseStructure: """All error responses from handle_serverless must follow the same schema. AI might return bare strings, dicts without 'error' key, etc.""" ERROR_CASES = [ - ({}, 400), # Missing reasoner - ({"reasoner": "nonexistent"}, 404), # Unknown reasoner - ({"path": ""}, 400), # Empty path + ({}, 400), # Missing reasoner + ({"reasoner": "nonexistent"}, 404), # Unknown reasoner + ({"path": ""}, 400), # Empty path ] def test_all_error_responses_have_error_key(self): from agentfield.agent import Agent + agent = Agent(node_id="error-schema") for event, expected_code in self.ERROR_CASES: result = agent.handle_serverless(event) - assert "statusCode" in result, \ + assert "statusCode" in result, ( f"Missing statusCode in error response for {event}" - assert "body" in result, \ - f"Missing body in error response for {event}" + ) + assert "body" in result, f"Missing body in error response for {event}" if result["statusCode"] >= 400: - assert "error" in result["body"], \ - f"Error response body missing 'error' key for status " \ + assert "error" in result["body"], ( + f"Error response body missing 'error' key for status " f"{result['statusCode']}, event={event}, body={result['body']}" + ) diff --git a/sdk/python/tests/test_issue_589.py b/sdk/python/tests/test_issue_589.py index 191e31649..4cb8be830 100644 --- a/sdk/python/tests/test_issue_589.py +++ b/sdk/python/tests/test_issue_589.py @@ -211,4 +211,4 @@ async def test_openrouter_subdomain_gets_auth_header(self, monkeypatch): ) assert isinstance(result, MultimodalResponse) - assert result.has_videos \ No newline at end of file + assert result.has_videos diff --git a/sdk/python/tests/test_media_providers_additional.py b/sdk/python/tests/test_media_providers_additional.py index 4ca792bd6..42dfd998f 100644 --- a/sdk/python/tests/test_media_providers_additional.py +++ b/sdk/python/tests/test_media_providers_additional.py @@ -30,10 +30,14 @@ async def generate_audio(self, text, **kwargs): def test_media_provider_generate_video_and_register_validation(): provider = DummyProvider() - with pytest.raises(NotImplementedError, match="dummy does not support video generation"): + with pytest.raises( + NotImplementedError, match="dummy does not support video generation" + ): __import__("asyncio").run(provider.generate_video("prompt")) - with pytest.raises(TypeError, match="provider_class must be a MediaProvider subclass"): + with pytest.raises( + TypeError, match="provider_class must be a MediaProvider subclass" + ): register_provider("bad", object) diff --git a/sdk/python/tests/test_memory_bigfiles_coverage.py b/sdk/python/tests/test_memory_bigfiles_coverage.py index fb2536fb8..23c078eee 100644 --- a/sdk/python/tests/test_memory_bigfiles_coverage.py +++ b/sdk/python/tests/test_memory_bigfiles_coverage.py @@ -40,7 +40,9 @@ def memory_client(execution_context): @pytest.mark.asyncio -async def test_async_request_uses_httpx_then_requests_fallback(monkeypatch, execution_context): +async def test_async_request_uses_httpx_then_requests_fallback( + monkeypatch, execution_context +): captured = {} class FakeAsyncClient: @@ -83,19 +85,25 @@ async def fake_to_thread(func, method, url, **kwargs): monkeypatch.setattr(builtins, "__import__", fake_import) monkeypatch.setattr("agentfield.memory._to_thread", fake_to_thread) - result = await client._async_request("POST", "http://example.test/b", json={"ok": True}) + result = await client._async_request( + "POST", "http://example.test/b", json={"ok": True} + ) assert result == "requests-response" assert captured["requests"][1] == "POST" @pytest.mark.asyncio -async def test_set_vector_delete_vector_list_keys_and_similarity(memory_client, monkeypatch): +async def test_set_vector_delete_vector_list_keys_and_similarity( + memory_client, monkeypatch +): calls = [] async def fake_request(method, url, **kwargs): calls.append((method, url, kwargs)) if url.endswith("/memory/list"): - return DummyResponse(payload=[{"key": "alpha"}, {"key": "beta"}, {"ignore": True}]) + return DummyResponse( + payload=[{"key": "alpha"}, {"key": "beta"}, {"ignore": True}] + ) if url.endswith("/memory/vector/search"): return DummyResponse(payload=[{"key": "alpha", "score": 0.9}]) return DummyResponse() @@ -129,7 +137,9 @@ async def fake_request(method, url, **kwargs): @pytest.mark.asyncio async def test_memory_operations_wrap_errors(memory_client, monkeypatch): - monkeypatch.setattr(memory_client, "_async_request", AsyncMock(side_effect=RuntimeError("boom"))) + monkeypatch.setattr( + memory_client, "_async_request", AsyncMock(side_effect=RuntimeError("boom")) + ) with pytest.raises(MemoryAccessError, match="Failed to delete memory key 'alpha'"): await memory_client.delete("alpha") @@ -140,7 +150,9 @@ async def test_memory_operations_wrap_errors(memory_client, monkeypatch): with pytest.raises(MemoryAccessError, match="Failed to delete vector key 'vec'"): await memory_client.delete_vector("vec") - with pytest.raises(MemoryAccessError, match="Failed to list keys for scope 'global'"): + with pytest.raises( + MemoryAccessError, match="Failed to list keys for scope 'global'" + ): await memory_client.list_keys("global") with pytest.raises(MemoryAccessError, match="Failed to perform similarity search"): @@ -149,13 +161,19 @@ async def test_memory_operations_wrap_errors(memory_client, monkeypatch): @pytest.mark.asyncio async def test_exists_and_scoped_clients_delegate(memory_client, monkeypatch): - monkeypatch.setattr(memory_client, "get", AsyncMock(side_effect=[{"ok": True}, RuntimeError("missing")])) + monkeypatch.setattr( + memory_client, + "get", + AsyncMock(side_effect=[{"ok": True}, RuntimeError("missing")]), + ) monkeypatch.setattr(memory_client, "set", AsyncMock()) monkeypatch.setattr(memory_client, "delete", AsyncMock()) monkeypatch.setattr(memory_client, "list_keys", AsyncMock(return_value=["k1"])) monkeypatch.setattr(memory_client, "set_vector", AsyncMock()) monkeypatch.setattr(memory_client, "delete_vector", AsyncMock()) - monkeypatch.setattr(memory_client, "similarity_search", AsyncMock(return_value=[{"id": 1}])) + monkeypatch.setattr( + memory_client, "similarity_search", AsyncMock(return_value=[{"id": 1}]) + ) assert await memory_client.exists("present") is True assert await memory_client.exists("missing") is False @@ -171,8 +189,12 @@ async def test_exists_and_scoped_clients_delegate(memory_client, monkeypatch): assert await scoped.similarity_search([1, 2]) == [{"id": 1}] assert await global_client.list_keys() == ["k1"] - memory_client.set.assert_awaited_once_with("name", {"value": 1}, scope="actor", scope_id="actor-1") - memory_client.delete.assert_awaited_once_with("name", scope="actor", scope_id="actor-1") + memory_client.set.assert_awaited_once_with( + "name", {"value": 1}, scope="actor", scope_id="actor-1" + ) + memory_client.delete.assert_awaited_once_with( + "name", scope="actor", scope_id="actor-1" + ) memory_client.list_keys.assert_any_await("actor", scope_id="actor-1") memory_client.list_keys.assert_any_await("global") diff --git a/sdk/python/tests/test_memory_client_core.py b/sdk/python/tests/test_memory_client_core.py index b992dc06e..6064bc761 100644 --- a/sdk/python/tests/test_memory_client_core.py +++ b/sdk/python/tests/test_memory_client_core.py @@ -282,9 +282,7 @@ async def test_scoped_client_delegates(memory_client): await scoped.delete_vector("chunk") await scoped.similarity_search([0.2]) - base.set.assert_awaited_once_with( - "key", 1, scope="session", scope_id="abc" - ) + base.set.assert_awaited_once_with("key", 1, scope="session", scope_id="abc") base.get.assert_awaited_once_with( "key", default=None, scope="session", scope_id="abc" ) diff --git a/sdk/python/tests/test_memory_coverage_additions.py b/sdk/python/tests/test_memory_coverage_additions.py index 3051d4e84..7ec5454dd 100644 --- a/sdk/python/tests/test_memory_coverage_additions.py +++ b/sdk/python/tests/test_memory_coverage_additions.py @@ -5,7 +5,12 @@ import pytest -from agentfield.memory import GlobalMemoryClient, MemoryClient, MemoryInterface, _vector_to_list +from agentfield.memory import ( + GlobalMemoryClient, + MemoryClient, + MemoryInterface, + _vector_to_list, +) @pytest.fixture @@ -45,7 +50,9 @@ async def test_get_returns_raw_string_when_data_is_not_json(memory_client, monke raise_for_status=lambda: None, json=lambda: {"data": "plain-text"}, ) - monkeypatch.setattr(memory_client, "_async_request", AsyncMock(return_value=response)) + monkeypatch.setattr( + memory_client, "_async_request", AsyncMock(return_value=response) + ) result = await memory_client.get("greeting") @@ -114,4 +121,3 @@ def test_memory_interface_scope_helpers_and_global_scope(memory_client): assert actor_client.scope == "actor" assert workflow_client.scope == "workflow" assert global_client.event_client is events - diff --git a/sdk/python/tests/test_memory_events.py b/sdk/python/tests/test_memory_events.py index 02420ccb5..60ee18383 100644 --- a/sdk/python/tests/test_memory_events.py +++ b/sdk/python/tests/test_memory_events.py @@ -112,7 +112,9 @@ def __init__(self): async def fake_connect(url, **kwargs): record["url"] = url - record["headers"] = kwargs.get("additional_headers") or kwargs.get("extra_headers") + record["headers"] = kwargs.get("additional_headers") or kwargs.get( + "extra_headers" + ) return DummyWebSocket() async def fake_listen(self): @@ -198,7 +200,9 @@ async def fake_reconnect(self): # Give the background task a chance to start await asyncio.sleep(0.05) - assert reconnect_started.is_set(), "reconnect should have been started in background" + assert reconnect_started.is_set(), ( + "reconnect should have been started in background" + ) assert not client.is_listening diff --git a/sdk/python/tests/test_memory_events_additional.py b/sdk/python/tests/test_memory_events_additional.py index 55706b77d..9190541ca 100644 --- a/sdk/python/tests/test_memory_events_additional.py +++ b/sdk/python/tests/test_memory_events_additional.py @@ -39,9 +39,12 @@ def test_memory_event_client_is_connected_variants(execution_context): @pytest.mark.asyncio -async def test_listen_handles_connection_closed_and_generic_errors(monkeypatch, execution_context): +async def test_listen_handles_connection_closed_and_generic_errors( + monkeypatch, execution_context +): client = MemoryEventClient("http://agentfield", execution_context) reconnect_calls = [] + async def fake_reconnect(): reconnect_calls.append("reconnect") @@ -56,7 +59,9 @@ class ClosedError(Exception): SimpleNamespace(ConnectionClosed=ClosedError), raising=False, ) - client.websocket = SimpleNamespace(recv=lambda: (_ for _ in ()).throw(ClosedError()), open=True) + client.websocket = SimpleNamespace( + recv=lambda: (_ for _ in ()).throw(ClosedError()), open=True + ) client.is_listening = True await client._listen() assert client.websocket is None @@ -77,7 +82,11 @@ async def close(self): socket = ClosingSocket() client.websocket = socket client.is_listening = True - monkeypatch.setattr("agentfield.memory_events.log_error", lambda message: reconnect_calls.append(message)) + monkeypatch.setattr( + "agentfield.memory_events.log_error", + lambda message: reconnect_calls.append(message), + ) + async def fake_retry(): reconnect_calls.append("retry") @@ -86,22 +95,32 @@ async def fake_retry(): assert socket.closed is True assert client.websocket is None - assert any("Error in event listener: bad recv" == entry for entry in reconnect_calls) + assert any( + "Error in event listener: bad recv" == entry for entry in reconnect_calls + ) assert "retry" in reconnect_calls @pytest.mark.asyncio -async def test_handle_reconnect_logs_max_and_failed_reconnect(monkeypatch, execution_context): +async def test_handle_reconnect_logs_max_and_failed_reconnect( + monkeypatch, execution_context +): client = MemoryEventClient("http://agentfield", execution_context) logged = [] monkeypatch.setattr("agentfield.memory_events.log_error", logged.append) - monkeypatch.setattr("agentfield.memory_events.log_info", lambda message: logged.append(message)) + monkeypatch.setattr( + "agentfield.memory_events.log_info", lambda message: logged.append(message) + ) original_sleep = asyncio.sleep - monkeypatch.setattr("agentfield.memory_events.asyncio.sleep", lambda delay: original_sleep(0)) + monkeypatch.setattr( + "agentfield.memory_events.asyncio.sleep", lambda delay: original_sleep(0) + ) client._reconnect_attempts = client._max_reconnect_attempts await client._handle_reconnect() - assert logged == [f"Max reconnection attempts reached ({client._max_reconnect_attempts})"] + assert logged == [ + f"Max reconnection attempts reached ({client._max_reconnect_attempts})" + ] logged.clear() client._reconnect_attempts = 0 @@ -116,7 +135,9 @@ async def failing_connect(): @pytest.mark.asyncio -async def test_subscribe_on_change_close_and_scoped_client(monkeypatch, execution_context): +async def test_subscribe_on_change_close_and_scoped_client( + monkeypatch, execution_context +): client = MemoryEventClient("http://agentfield", execution_context) scheduled = [] @@ -125,7 +146,9 @@ def fake_create_task(coro): coro.close() return SimpleNamespace() - monkeypatch.setattr("agentfield.memory_events.asyncio.create_task", fake_create_task) + monkeypatch.setattr( + "agentfield.memory_events.asyncio.create_task", fake_create_task + ) async def callback(event): return event.key @@ -138,7 +161,9 @@ async def callback(event): async def wrapped(event): return event.scope_id - event = MemoryChangeEvent(scope="session", scope_id="s1", key="order.id", action="set") + event = MemoryChangeEvent( + scope="session", scope_id="s1", key="order.id", action="set" + ) assert await wrapped(event) == "s1" assert wrapped._memory_event_listener is True assert wrapped._memory_event_patterns == ["order.*"] @@ -177,7 +202,9 @@ async def fake_history(**kwargs): @pytest.mark.asyncio -async def test_history_handles_parse_errors_importerror_fallback_and_request_failures(monkeypatch, execution_context): +async def test_history_handles_parse_errors_importerror_fallback_and_request_failures( + monkeypatch, execution_context +): client = MemoryEventClient("http://agentfield", execution_context, api_key="secret") logged = [] monkeypatch.setattr("agentfield.memory_events.log_error", logged.append) @@ -188,7 +215,12 @@ def raise_for_status(self): def json(self): return [ - {"scope": "session", "scope_id": "s1", "key": "cart.total", "action": "set"}, + { + "scope": "session", + "scope_id": "s1", + "key": "cart.total", + "action": "set", + }, [], ] @@ -232,7 +264,14 @@ def raise_for_status(self): return None def json(self): - return [{"scope": "agent", "scope_id": "a1", "key": "profile.name", "action": "set"}] + return [ + { + "scope": "agent", + "scope_id": "a1", + "key": "profile.name", + "action": "set", + } + ] requests_module = SimpleNamespace( get=lambda url, params=None, headers=None, timeout=None: RequestsResponse() diff --git a/sdk/python/tests/test_memory_invariants.py b/sdk/python/tests/test_memory_invariants.py index 8fa725bf9..33d4a885e 100644 --- a/sdk/python/tests/test_memory_invariants.py +++ b/sdk/python/tests/test_memory_invariants.py @@ -5,6 +5,7 @@ hold regardless of implementation changes. They use mocks to isolate the client from HTTP and verify behavioral contracts at the API boundary. """ + from __future__ import annotations from typing import Any @@ -17,6 +18,7 @@ # Helpers # --------------------------------------------------------------------------- + def _make_memory_client(api_base: str = "http://localhost:8080/api/v1"): """Build a MemoryClient with a fully mocked AgentFieldClient and ExecutionContext.""" from agentfield.memory import MemoryClient @@ -62,6 +64,7 @@ def _not_found_response(): # 1. Scope hierarchy order # --------------------------------------------------------------------------- + class TestScopeHierarchyOrder: """ The hierarchical lookup MUST check workflow → session → actor → global. @@ -130,6 +133,7 @@ async def test_invariant_scope_override_sends_scope_in_payload(self): # 2. Scope independence # --------------------------------------------------------------------------- + class TestScopeIndependence: """Setting a key in one scope must not affect the same key in another scope.""" @@ -141,7 +145,9 @@ async def test_invariant_workflow_scope_set_uses_different_header_than_global(se """ client = _make_memory_client() client.agentfield_client._async_request.return_value = _ok_response() - client.agentfield_client._async_request.return_value.raise_for_status = MagicMock() + client.agentfield_client._async_request.return_value.raise_for_status = ( + MagicMock() + ) # Call with workflow scope override await client.set("x", "workflow-value", scope="workflow", scope_id="wf-123") @@ -166,7 +172,9 @@ async def test_invariant_global_scope_set_does_not_inject_workflow_header(self): inner_client = _make_memory_client() inner_client.agentfield_client._async_request.return_value = _ok_response() - inner_client.agentfield_client._async_request.return_value.raise_for_status = MagicMock() + inner_client.agentfield_client._async_request.return_value.raise_for_status = ( + MagicMock() + ) global_client = GlobalMemoryClient(inner_client) await global_client.set("x", "global-value") @@ -189,7 +197,9 @@ async def test_invariant_scoped_clients_send_different_scope_ids(self): client = _make_memory_client() client.agentfield_client._async_request.return_value = _ok_response() - client.agentfield_client._async_request.return_value.raise_for_status = MagicMock() + client.agentfield_client._async_request.return_value.raise_for_status = ( + MagicMock() + ) wf_a = ScopedMemoryClient(client, "workflow", "wf-AAA") wf_b = ScopedMemoryClient(client, "workflow", "wf-BBB") @@ -217,6 +227,7 @@ async def test_invariant_scoped_clients_send_different_scope_ids(self): # 3. Key namespacing # --------------------------------------------------------------------------- + class TestKeyNamespacing: """Two different keys in the same scope are fully independent.""" @@ -228,7 +239,9 @@ async def test_invariant_different_keys_produce_different_payloads(self): """ client = _make_memory_client() client.agentfield_client._async_request.return_value = _ok_response() - client.agentfield_client._async_request.return_value.raise_for_status = MagicMock() + client.agentfield_client._async_request.return_value.raise_for_status = ( + MagicMock() + ) await client.set("key-A", "value-1") call_a = client.agentfield_client._async_request.call_args @@ -272,6 +285,7 @@ async def test_invariant_get_passes_exact_requested_key(self): # 4. Null safety # --------------------------------------------------------------------------- + class TestNullSafety: """Getting a non-existent key must return None/default, never crash.""" diff --git a/sdk/python/tests/test_memory_performance.py b/sdk/python/tests/test_memory_performance.py index 9cfbf385d..1b5a637e1 100644 --- a/sdk/python/tests/test_memory_performance.py +++ b/sdk/python/tests/test_memory_performance.py @@ -24,6 +24,7 @@ @dataclass class MemoryMetrics: """Memory measurement results.""" + name: str peak_mb: float current_mb: float @@ -38,7 +39,11 @@ def per_iteration_kb(self) -> float: @property def reduction_pct(self) -> float: """Memory reduction from peak to current.""" - return ((self.peak_mb - self.current_mb) / self.peak_mb * 100) if self.peak_mb > 0 else 0 + return ( + ((self.peak_mb - self.current_mb) / self.peak_mb * 100) + if self.peak_mb > 0 + else 0 + ) def measure_memory(func, iterations: int = 1000) -> MemoryMetrics: @@ -83,14 +88,16 @@ def test_cleanup_interval_is_aggressive(self): def test_completed_execution_retention_is_short(self): """Completed execution retention should be short.""" config = AsyncConfig() - assert config.completed_execution_retention_seconds <= 120.0, \ + assert config.completed_execution_retention_seconds <= 120.0, ( "Retention should be <= 120s" + ) def test_max_completed_executions_is_bounded(self): """Max completed executions should be bounded.""" config = AsyncConfig() - assert config.max_completed_executions <= 2000, \ + assert config.max_completed_executions <= 2000, ( "Max completed executions should be <= 2000" + ) class TestExecutionStateMemory: @@ -106,7 +113,7 @@ def _create_execution_states(self, count: int) -> List[ExecutionState]: input_data={ "payload": "x" * 10000, # ~10KB "nested": {"items": list(range(500))}, - } + }, ) states.append(state) return states @@ -116,7 +123,7 @@ def test_input_data_cleared_on_success(self): state = ExecutionState( execution_id="test_exec", target="agent.reasoner", - input_data={"large": "x" * 10000} + input_data={"large": "x" * 10000}, ) assert len(state.input_data) > 0 @@ -130,7 +137,7 @@ def test_input_data_cleared_on_error(self): state = ExecutionState( execution_id="test_exec", target="agent.reasoner", - input_data={"large": "x" * 10000} + input_data={"large": "x" * 10000}, ) state.set_error("Test error") @@ -143,7 +150,7 @@ def test_input_data_cleared_on_cancel(self): state = ExecutionState( execution_id="test_exec", target="agent.reasoner", - input_data={"large": "x" * 10000} + input_data={"large": "x" * 10000}, ) state.cancel("User cancelled") @@ -157,7 +164,7 @@ def test_input_data_cleared_on_timeout(self): execution_id="test_exec", target="agent.reasoner", input_data={"large": "x" * 10000}, - timeout=1.0 + timeout=1.0, ) state.timeout_execution() @@ -167,6 +174,7 @@ def test_input_data_cleared_on_timeout(self): def test_memory_reduction_after_completion(self): """Memory should be significantly reduced after executions complete.""" + def benchmark(iterations: int): states = self._create_execution_states(iterations) # Complete 70% of executions @@ -177,8 +185,9 @@ def benchmark(iterations: int): metrics = measure_memory(benchmark, iterations=1000) # Memory should be reduced by at least 50% after clearing input_data - assert metrics.reduction_pct >= 50.0, \ + assert metrics.reduction_pct >= 50.0, ( f"Expected >= 50% memory reduction, got {metrics.reduction_pct:.1f}%" + ) class TestResultCacheMemory: @@ -193,11 +202,13 @@ def test_cache_respects_max_size(self): for i in range(config.result_cache_max_size + 1000): cache.set(f"key_{i}", {"data": "x" * 100}) - assert len(cache) <= config.result_cache_max_size, \ + assert len(cache) <= config.result_cache_max_size, ( "Cache should not exceed max size" + ) def test_cache_memory_is_bounded(self): """Cache memory should be bounded by max size.""" + def benchmark(iterations: int): config = AsyncConfig() cache = ResultCache(config) @@ -208,8 +219,9 @@ def benchmark(iterations: int): metrics = measure_memory(benchmark, iterations=10000) # With 5000 max entries at ~1KB each, should be under 10MB - assert metrics.current_mb < 10.0, \ + assert metrics.current_mb < 10.0, ( f"Cache memory should be bounded, got {metrics.current_mb:.2f} MB" + ) class TestClientSessionReuse: @@ -222,8 +234,9 @@ def test_shared_session_is_created(self): AgentFieldClient(base_url="http://localhost:8080") # Creates shared session - assert AgentFieldClient._shared_sync_session is not None, \ + assert AgentFieldClient._shared_sync_session is not None, ( "Shared session should be created" + ) def test_multiple_clients_share_session(self): """Multiple clients should share the same sync session.""" @@ -253,8 +266,9 @@ def benchmark(iterations: int): metrics = measure_memory(benchmark, iterations=100) # 100 clients should use less than 1MB total - assert metrics.current_mb < 1.0, \ + assert metrics.current_mb < 1.0, ( f"Client creation should be memory efficient, got {metrics.current_mb:.2f} MB" + ) class TestMemoryBenchmarkBaseline: @@ -280,19 +294,24 @@ def _create_baseline_workload(self, iterations: int) -> List[Dict[str, Any]]: } # Simulate history accumulation (common pattern) for j in range(10): - workload["history"].append({ - "role": "user", - "content": f"Message {j}: " + "y" * 500, - }) - workload["history"].append({ - "role": "assistant", - "content": f"Response {j}: " + "z" * 500, - }) + workload["history"].append( + { + "role": "user", + "content": f"Message {j}: " + "y" * 500, + } + ) + workload["history"].append( + { + "role": "assistant", + "content": f"Response {j}: " + "z" * 500, + } + ) workloads.append(workload) return workloads def test_baseline_memory_usage(self): """Measure baseline memory usage for comparison.""" + def benchmark(iterations: int): return self._create_baseline_workload(iterations) @@ -313,6 +332,7 @@ def benchmark(iterations: int): def test_optimized_sdk_memory_usage(self): """Measure optimized SDK memory usage.""" + def benchmark(iterations: int): config = AsyncConfig() cache = ResultCache(config) @@ -326,7 +346,7 @@ def benchmark(iterations: int): "payload": "x" * 10000, "nested": {"items": list(range(500))}, "metadata": {"id": f"run_{i}"}, - } + }, ) state.set_result({"output": f"result_{i}"}) cache.set_execution_result(state.execution_id, state.result) @@ -348,10 +368,12 @@ def benchmark(iterations: int): print(f"{'=' * 60}") # Optimized SDK should use significantly less memory - assert metrics.current_mb < 10.0, \ + assert metrics.current_mb < 10.0, ( f"Optimized SDK should use < 10MB, got {metrics.current_mb:.2f} MB" - assert metrics.per_iteration_kb < 10.0, \ + ) + assert metrics.per_iteration_kb < 10.0, ( f"Per-iteration memory should be < 10KB, got {metrics.per_iteration_kb:.2f} KB" + ) @pytest.fixture @@ -367,7 +389,9 @@ def memory_report(): print(f"{'Test Name':<40} {'Current':>10} {'Peak':>10} {'Per Iter':>12}") print("-" * 70) for m in metrics_list: - print(f"{m.name:<40} {m.current_mb:>8.2f}MB {m.peak_mb:>8.2f}MB {m.per_iteration_kb:>10.2f}KB") + print( + f"{m.name:<40} {m.current_mb:>8.2f}MB {m.peak_mb:>8.2f}MB {m.per_iteration_kb:>10.2f}KB" + ) print("=" * 70) @@ -383,9 +407,7 @@ def exec_state_benchmark(n): states = [] for i in range(n): s = ExecutionState( - execution_id=f"e_{i}", - target="a.r", - input_data={"p": "x" * 10000} + execution_id=f"e_{i}", target="a.r", input_data={"p": "x" * 10000} ) s.set_result({"o": i}) states.append(s) @@ -412,7 +434,9 @@ def cache_benchmark(n): def client_benchmark(n): clients = [] for i in range(n): - clients.append(AgentFieldClient(base_url=f"http://localhost:808{i%10}")) + clients.append( + AgentFieldClient(base_url=f"http://localhost:808{i % 10}") + ) return clients m3 = measure_memory(client_benchmark, 100) diff --git a/sdk/python/tests/test_multimodal_response_additional.py b/sdk/python/tests/test_multimodal_response_additional.py index 4eab77273..3ac4a1536 100644 --- a/sdk/python/tests/test_multimodal_response_additional.py +++ b/sdk/python/tests/test_multimodal_response_additional.py @@ -31,8 +31,12 @@ def make_module(content): return make_module -def test_image_and_file_outputs_support_url_downloads(tmp_path, monkeypatch, fake_requests_module): - monkeypatch.setitem(__import__("sys").modules, "requests", fake_requests_module(b"remote")) +def test_image_and_file_outputs_support_url_downloads( + tmp_path, monkeypatch, fake_requests_module +): + monkeypatch.setitem( + __import__("sys").modules, "requests", fake_requests_module(b"remote") + ) image = ImageOutput(url="https://example.com/test.png") image_path = tmp_path / "image.png" @@ -70,8 +74,12 @@ def test_output_objects_raise_for_missing_data(): def test_audio_play_and_image_show_log_failures(monkeypatch): logged = {"warn": [], "error": []} - monkeypatch.setattr("agentfield.multimodal_response.log_warn", logged["warn"].append) - monkeypatch.setattr("agentfield.multimodal_response.log_error", logged["error"].append) + monkeypatch.setattr( + "agentfield.multimodal_response.log_warn", logged["warn"].append + ) + monkeypatch.setattr( + "agentfield.multimodal_response.log_error", logged["error"].append + ) original_import = builtins.__import__ @@ -104,7 +112,9 @@ def open(_): def test_multimodal_response_save_all_and_flags(tmp_path): audio = AudioOutput(data=base64.b64encode(b"aud").decode(), format="wav") - image = ImageOutput(b64_json=base64.b64encode(b"img").decode(), url="https://cdn/test.jpg") + image = ImageOutput( + b64_json=base64.b64encode(b"img").decode(), url="https://cdn/test.jpg" + ) file_output = FileOutput( data=base64.b64encode(b"file").decode(), filename="artifact.bin", @@ -169,8 +179,13 @@ def broken(self): raise RuntimeError("ignore me") found = _find_images_recursive([Nested(), {"other": direct_image}], max_depth=5) - assert [img.url for img in found] == ["https://example.com/b.png", "https://example.com/a.png"] - assert _find_images_recursive({"url": "https://example.com/c.png"}, max_depth=0) == [] + assert [img.url for img in found] == [ + "https://example.com/b.png", + "https://example.com/a.png", + ] + assert ( + _find_images_recursive({"url": "https://example.com/c.png"}, max_depth=0) == [] + ) def test_detect_multimodal_response_completion_and_cost(monkeypatch): @@ -198,13 +213,21 @@ def test_detect_multimodal_response_completion_and_cost(monkeypatch): assert result.text == "hello" assert result.audio.get_bytes() == b"aud" assert result.images[0].b64_json == "Zm9v" - assert result.usage == {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + assert result.usage == { + "prompt_tokens": 1, + "completion_tokens": 2, + "total_tokens": 3, + } assert result.cost_usd == 1.25 def test_detect_multimodal_response_other_shapes_and_fallbacks(): image_generation = SimpleNamespace( - data=[SimpleNamespace(url="https://example.com/generated.png", revised_prompt="prompt")] + data=[ + SimpleNamespace( + url="https://example.com/generated.png", revised_prompt="prompt" + ) + ] ) tts_response = SimpleNamespace( text="spoken", diff --git a/sdk/python/tests/test_multimodal_response_cost.py b/sdk/python/tests/test_multimodal_response_cost.py index 755645288..bf3d435d5 100644 --- a/sdk/python/tests/test_multimodal_response_cost.py +++ b/sdk/python/tests/test_multimodal_response_cost.py @@ -1,8 +1,12 @@ """Functional tests for cost/usage extraction in MultimodalResponse.""" + import types from unittest.mock import MagicMock, patch -from agentfield.multimodal_response import MultimodalResponse, detect_multimodal_response +from agentfield.multimodal_response import ( + MultimodalResponse, + detect_multimodal_response, +) class TestMultimodalResponseCostFields: @@ -73,9 +77,7 @@ def test_extracts_cost_when_litellm_available(self): with patch.dict("sys.modules", {"litellm": mock_litellm}): result = detect_multimodal_response(resp) assert result.cost_usd == 0.0035 - mock_litellm.completion_cost.assert_called_once_with( - completion_response=resp - ) + mock_litellm.completion_cost.assert_called_once_with(completion_response=resp) def test_cost_is_none_when_litellm_raises(self): resp = _make_litellm_response() @@ -96,9 +98,7 @@ def test_no_usage_when_response_lacks_usage_field(self): resp = types.SimpleNamespace( choices=[ types.SimpleNamespace( - message=types.SimpleNamespace( - content="hi", audio=None, images=None - ) + message=types.SimpleNamespace(content="hi", audio=None, images=None) ) ], model="gpt-4o", @@ -116,9 +116,7 @@ def test_no_cost_when_response_lacks_model(self): resp = types.SimpleNamespace( choices=[ types.SimpleNamespace( - message=types.SimpleNamespace( - content="hi", audio=None, images=None - ) + message=types.SimpleNamespace(content="hi", audio=None, images=None) ) ], model=None, diff --git a/sdk/python/tests/test_pydantic_utils.py b/sdk/python/tests/test_pydantic_utils.py index c5fd90c81..262d2ca7b 100644 --- a/sdk/python/tests/test_pydantic_utils.py +++ b/sdk/python/tests/test_pydantic_utils.py @@ -117,9 +117,7 @@ def test_convert_validation_error_propagation(): def my_func(m: MyModel): return m - _, kwargs = convert_function_args( - my_func, (), {"m": {"x": "not-an-int"}} - ) + _, kwargs = convert_function_args(my_func, (), {"m": {"x": "not-an-int"}}) # Current behavior: current implementation swallows the exception due to incompatibility with Pydantic v2 (ValidationError constructor signature mismatch), and returns original args - assert kwargs["m"] == {"x": "not-an-int"} \ No newline at end of file + assert kwargs["m"] == {"x": "not-an-int"} diff --git a/sdk/python/tests/test_reasoner_path_normalization.py b/sdk/python/tests/test_reasoner_path_normalization.py index 1357c1bc7..23805ed1a 100644 --- a/sdk/python/tests/test_reasoner_path_normalization.py +++ b/sdk/python/tests/test_reasoner_path_normalization.py @@ -113,7 +113,9 @@ async def handler(input_data: dict, _idx=idx) -> dict: json={"input_data": {}}, headers={"x-workflow-id": "wf-dyn", "x-execution-id": f"exec-dyn-{i}"}, ) - assert resp.status_code == 200, f"handler-{i} not reachable at /reasoners/handler-{i}" + assert resp.status_code == 200, ( + f"handler-{i} not reachable at /reasoners/handler-{i}" + ) @pytest.mark.asyncio diff --git a/sdk/python/tests/test_result_cache_bigfiles_coverage.py b/sdk/python/tests/test_result_cache_bigfiles_coverage.py index 926cad708..bce1ac329 100644 --- a/sdk/python/tests/test_result_cache_bigfiles_coverage.py +++ b/sdk/python/tests/test_result_cache_bigfiles_coverage.py @@ -26,7 +26,9 @@ def test_cache_entry_and_metrics_properties(monkeypatch): def test_cache_contains_delete_clear_repr_and_execution_helpers(monkeypatch): - cfg = AsyncConfig(enable_result_caching=True, result_cache_ttl=30.0, result_cache_max_size=3) + cfg = AsyncConfig( + enable_result_caching=True, result_cache_ttl=30.0, result_cache_max_size=3 + ) cache = ResultCache(cfg) cache.set_execution_result("exec-1", {"ok": True}) @@ -86,8 +88,12 @@ def test_cleanup_expired_and_disabled_cache_behaviour(monkeypatch): assert cache.metrics.misses == 1 enabled = ResultCache(AsyncConfig(enable_result_caching=True, result_cache_ttl=1.0)) - enabled._cache["old"] = CacheEntry(value=1, created_at=0.0, accessed_at=0.0, ttl=0.5) - enabled._cache["new"] = CacheEntry(value=2, created_at=10.0, accessed_at=10.0, ttl=5.0) + enabled._cache["old"] = CacheEntry( + value=1, created_at=0.0, accessed_at=0.0, ttl=0.5 + ) + enabled._cache["new"] = CacheEntry( + value=2, created_at=10.0, accessed_at=10.0, ttl=5.0 + ) monkeypatch.setattr("agentfield.result_cache.time.time", lambda: 2.0) removed = enabled._cleanup_expired() @@ -124,8 +130,12 @@ def fake_cleanup(): monkeypatch.setattr("agentfield.result_cache.asyncio.sleep", fake_sleep) monkeypatch.setattr(cache, "_cleanup_expired", fake_cleanup) - monkeypatch.setattr("agentfield.result_cache.logger.debug", lambda msg: debug_messages.append(msg)) - monkeypatch.setattr("agentfield.result_cache.logger.error", lambda msg: error_messages.append(msg)) + monkeypatch.setattr( + "agentfield.result_cache.logger.debug", lambda msg: debug_messages.append(msg) + ) + monkeypatch.setattr( + "agentfield.result_cache.logger.error", lambda msg: error_messages.append(msg) + ) await cache.start() await cache._cleanup_task diff --git a/sdk/python/tests/test_simulate_trigger.py b/sdk/python/tests/test_simulate_trigger.py index fa5f71a77..de443436c 100644 --- a/sdk/python/tests/test_simulate_trigger.py +++ b/sdk/python/tests/test_simulate_trigger.py @@ -167,9 +167,7 @@ def handler(input, webhook: TriggerContext): return webhook.event_type bound = _make_reasoner(handler, [EventTrigger(source="slack")]) - out = simulate_trigger( - bound, source="slack", event_type="app_mention", body={} - ) + out = simulate_trigger(bound, source="slack", event_type="app_mention", body={}) assert out == "app_mention" def test_injects_ctx_with_trigger(self): @@ -199,10 +197,7 @@ async def handler(input, trigger: TriggerContext): return f"{trigger.source}:{input['n']}" bound = _make_reasoner(handler, [EventTrigger(source="github")]) - assert ( - simulate_trigger(bound, source="github", body={"n": 7}) - == "github:7" - ) + assert simulate_trigger(bound, source="github", body={"n": 7}) == "github:7" class TestSimulateSchedule: diff --git a/sdk/python/tests/test_strict_openai_schema.py b/sdk/python/tests/test_strict_openai_schema.py index cde13f3cf..502b2fd48 100644 --- a/sdk/python/tests/test_strict_openai_schema.py +++ b/sdk/python/tests/test_strict_openai_schema.py @@ -19,8 +19,12 @@ def _assert_strict(node): """Recursively assert every object node is OpenAI-strict-compliant.""" if isinstance(node, dict): props = node.get("properties") - if isinstance(props, dict) and (node.get("type") == "object" or "type" not in node): - assert node.get("additionalProperties") is False, f"missing additionalProperties:false in {node}" + if isinstance(props, dict) and ( + node.get("type") == "object" or "type" not in node + ): + assert node.get("additionalProperties") is False, ( + f"missing additionalProperties:false in {node}" + ) assert set(node.get("required", [])) == set(props.keys()), ( f"required {node.get('required')} != properties {list(props.keys())}" ) diff --git a/sdk/python/tests/test_tool_calling.py b/sdk/python/tests/test_tool_calling.py index 107aa8921..93d288061 100644 --- a/sdk/python/tests/test_tool_calling.py +++ b/sdk/python/tests/test_tool_calling.py @@ -609,13 +609,22 @@ async def mock_completion(params): class TestInvocationTargetConversion: def test_skill_format(self): - assert _invocation_target_to_call_target("utility-worker:skill:get_weather") == "utility-worker.get_weather" + assert ( + _invocation_target_to_call_target("utility-worker:skill:get_weather") + == "utility-worker.get_weather" + ) def test_reasoner_format(self): - assert _invocation_target_to_call_target("utility-worker:summarize") == "utility-worker.summarize" + assert ( + _invocation_target_to_call_target("utility-worker:summarize") + == "utility-worker.summarize" + ) def test_dot_format_passthrough(self): - assert _invocation_target_to_call_target("utility-worker.summarize") == "utility-worker.summarize" + assert ( + _invocation_target_to_call_target("utility-worker.summarize") + == "utility-worker.summarize" + ) def test_no_separator(self): assert _invocation_target_to_call_target("standalone") == "standalone" diff --git a/sdk/python/tests/test_tool_calling_error_paths.py b/sdk/python/tests/test_tool_calling_error_paths.py index b00f62f33..c3d6c84c0 100644 --- a/sdk/python/tests/test_tool_calling_error_paths.py +++ b/sdk/python/tests/test_tool_calling_error_paths.py @@ -22,7 +22,10 @@ def make_tool_schema(name: str = "utility.echo"): "function": { "name": name, "description": "Echo text", - "parameters": {"type": "object", "properties": {"text": {"type": "string"}}}, + "parameters": { + "type": "object", + "properties": {"text": {"type": "string"}}, + }, }, } @@ -238,7 +241,9 @@ async def test_missing_tool_is_reported_back_to_llm(): make_completion = AsyncMock( side_effect=[ make_llm_response( - tool_calls=[make_tool_call(name="utility.missing", arguments='{"x": 1}')] + tool_calls=[ + make_tool_call(name="utility.missing", arguments='{"x": 1}') + ] ), make_llm_response(content="Missing tool handled"), ] diff --git a/sdk/python/tests/test_trigger_context.py b/sdk/python/tests/test_trigger_context.py index e979fa40d..a908f172e 100644 --- a/sdk/python/tests/test_trigger_context.py +++ b/sdk/python/tests/test_trigger_context.py @@ -24,7 +24,7 @@ def test_trigger_context_creation(self): idempotency_key="evt_abc_key", received_at=now, ) - + assert ctx.trigger_id == "trg_stripe_123" assert ctx.source == "stripe" assert ctx.event_type == "payment_intent.succeeded" @@ -45,7 +45,7 @@ def test_trigger_context_with_vc_id(self): received_at=datetime.utcnow(), vc_id="vc_webhook_123", ) - + assert ctx.vc_id == "vc_webhook_123" @pytest.mark.unit @@ -59,7 +59,7 @@ def test_trigger_context_frozen(self): idempotency_key="key_1", received_at=datetime.utcnow(), ) - + with pytest.raises(AttributeError): ctx.source = "github" @@ -85,7 +85,7 @@ def test_execution_context_trigger_field(self): reasoner_name="handler", trigger=trigger, ) - + assert ctx.trigger is trigger assert ctx.trigger.source == "stripe" @@ -98,7 +98,7 @@ def test_execution_context_trigger_none_by_default(self): agent_instance=None, reasoner_name="handler", ) - + assert ctx.trigger is None @pytest.mark.unit @@ -119,9 +119,9 @@ def test_child_context_inherits_trigger(self): reasoner_name="parent", trigger=trigger, ) - + child = parent.child_context() - + assert child.trigger is trigger assert child.execution_id != parent.execution_id assert child.parent_execution_id == parent.execution_id @@ -133,15 +133,16 @@ class TestEventTriggerTransform: @pytest.mark.unit def test_event_trigger_with_transform(self): """Verify EventTrigger accepts sync transform callable.""" + def stripe_to_order(event_dict): return {"order_id": event_dict.get("id")} - + trigger = EventTrigger( source="stripe", types=["payment_intent.succeeded"], transform=stripe_to_order, ) - + assert trigger.transform is stripe_to_order @pytest.mark.unit @@ -153,9 +154,10 @@ def test_event_trigger_transform_default_none(self): @pytest.mark.unit def test_event_trigger_rejects_async_transform(self): """Verify EventTrigger raises TypeError for async transform.""" + async def bad_transform(event): return event - + with pytest.raises(TypeError, match="must be sync"): EventTrigger( source="stripe", @@ -165,15 +167,16 @@ async def bad_transform(event): @pytest.mark.unit def test_event_trigger_transform_not_in_comparison(self): """Verify transform field is excluded from equality/repr.""" + def transform1(e): return e - + def transform2(e): return e - + t1 = EventTrigger(source="stripe", transform=transform1) t2 = EventTrigger(source="stripe", transform=transform2) - + # Should be equal because transform is excluded from comparison assert t1 == t2 # And not in repr @@ -182,7 +185,7 @@ def transform2(e): class TestEnvelopeUnwrapping: """Test dispatcher envelope detection and unwrapping. - + Note: These tests use the agent's _detect_and_unwrap_trigger_envelope method which is tested indirectly through integration with _execute_reasoner_endpoint. """ @@ -202,7 +205,7 @@ def test_envelope_structure(self): "vc_id": "vc_123", }, } - + # Verify envelope structure assert "event" in envelope assert "_meta" in envelope @@ -216,14 +219,15 @@ class TestTransformMatching: @pytest.mark.unit def test_transform_matching_exact_source(self): """Verify transform matches on source.""" + def stripe_transform(e): return {"transformed": True} - + binding = EventTrigger( source="stripe", transform=stripe_transform, ) - + # Binding source matches trigger source assert binding.source == "stripe" assert binding.transform is stripe_transform @@ -243,24 +247,25 @@ def test_transform_matching_event_type_prefix(self): @pytest.mark.unit def test_transform_specificity(self): """Verify most-specific binding is chosen.""" + def broad_transform(e): return {"binding": "broad"} - + def specific_transform(e): return {"binding": "specific"} - + broad = EventTrigger( source="stripe", types=[], # accepts all transform=broad_transform, ) - + specific = EventTrigger( source="stripe", types=["payment_intent.succeeded"], # specific transform=specific_transform, ) - + # Both match source, but specific should be preferred assert broad.source == specific.source assert len(broad.types) == 0 @@ -275,15 +280,13 @@ def test_trigger_context_passed_to_reasoner(self, test_agent): """Verify trigger context is passed to reasoner accepting trigger parameter.""" # This test requires a running agent; see conftest.py for setup captured_trigger = None - - @test_agent.reasoner( - triggers=[EventTrigger(source="stripe")] - ) + + @test_agent.reasoner(triggers=[EventTrigger(source="stripe")]) async def handle_webhook(input, trigger): nonlocal captured_trigger captured_trigger = trigger return {"ok": True} - + # Simulate envelope POST _envelope = { "event": {"id": "pi_123"}, @@ -308,12 +311,12 @@ class TestTransformExecution: def test_transform_called_on_match(self, test_agent): """Verify transform is called when event matches binding.""" transform_called = False - + def stripe_to_order(event_dict): nonlocal transform_called transform_called = True return {"order_id": event_dict.get("id")} - + @test_agent.reasoner( triggers=[ EventTrigger( @@ -325,7 +328,7 @@ def stripe_to_order(event_dict): ) async def handle_payment(input): return {"received": input} - + # Note: Real test would POST envelope @@ -337,7 +340,7 @@ def test_direct_call_no_envelope(self): """Verify non-envelope payloads work unchanged.""" # Direct input (not an envelope) payload = {"order_id": "123", "amount": 100} - + # Should not look like an envelope assert "event" not in payload or "_meta" not in payload @@ -349,5 +352,5 @@ def test_execution_context_trigger_none_on_direct_call(self): agent_instance=None, reasoner_name="reasoner", ) - + assert ctx.trigger is None diff --git a/sdk/python/tests/test_types.py b/sdk/python/tests/test_types.py index 4d68c8aff..ddb82397d 100644 --- a/sdk/python/tests/test_types.py +++ b/sdk/python/tests/test_types.py @@ -1,6 +1,7 @@ """ Tests for agentfield.types — data models, enums, and configuration classes. """ + from __future__ import annotations import asyncio @@ -20,7 +21,6 @@ ExecutionHeaders, HarnessConfig, HeartbeatData, - MemoryChangeEvent, MemoryConfig, MemoryValue, @@ -83,9 +83,7 @@ def test_to_dict_serializes_status_value(self): assert d["timestamp"] == "2024-01-01T00:00:00Z" def test_to_dict_offline(self): - hb = HeartbeatData( - status=AgentStatus.OFFLINE, timestamp="t" - ) + hb = HeartbeatData(status=AgentStatus.OFFLINE, timestamp="t") d = hb.to_dict() assert d["status"] == "offline" @@ -239,9 +237,7 @@ def test_defaults(self): class TestDiscoveryPagination: def test_from_dict(self): - dp = DiscoveryPagination.from_dict( - {"limit": 10, "offset": 5, "has_more": True} - ) + dp = DiscoveryPagination.from_dict({"limit": 10, "offset": 5, "has_more": True}) assert dp.limit == 10 assert dp.offset == 5 assert dp.has_more is True @@ -426,9 +422,7 @@ class TestCompactDiscoveryResponse: def test_from_dict(self): data = { "discovered_at": "2024-01-01", - "reasoners": [ - {"id": "r1", "agent_id": "a1", "target": "t", "tags": []} - ], + "reasoners": [{"id": "r1", "agent_id": "a1", "target": "t", "tags": []}], "skills": [], } cdr = CompactDiscoveryResponse.from_dict(data) diff --git a/sdk/python/tests/test_vc_generator.py b/sdk/python/tests/test_vc_generator.py index de1a1c9f4..2d8557af3 100644 --- a/sdk/python/tests/test_vc_generator.py +++ b/sdk/python/tests/test_vc_generator.py @@ -34,7 +34,8 @@ def test_generate_execution_vc_success(monkeypatch): "input_hash": "hash-in", "output_hash": "hash-out", "status": "succeeded", - "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + + "Z", } def fake_post(url, json=None, headers=None, timeout=None): @@ -79,8 +80,10 @@ def test_create_workflow_vc(monkeypatch): "component_vcs": ["vc-1"], "workflow_vc_id": "wvc-1", "status": "succeeded", - "start_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", - "end_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "start_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + + "Z", + "end_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + + "Z", "total_steps": 1, "completed_steps": 1, } diff --git a/sdk/python/tests/test_vc_generator_error_paths.py b/sdk/python/tests/test_vc_generator_error_paths.py index b9ea765cc..89aae9b59 100644 --- a/sdk/python/tests/test_vc_generator_error_paths.py +++ b/sdk/python/tests/test_vc_generator_error_paths.py @@ -35,9 +35,7 @@ def make_execution_payload() -> Dict[str, Any]: "input_hash": "hash-in", "output_hash": "hash-out", "status": "succeeded", - "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[ - :-3 - ] + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", } @@ -49,13 +47,9 @@ def make_workflow_payload() -> Dict[str, Any]: "component_vcs": ["vc-1"], "workflow_vc_id": "wvc-1", "status": "succeeded", - "start_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[ - :-3 - ] + "start_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", - "end_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[ - :-3 - ] + "end_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", "total_steps": 1, "completed_steps": 1, diff --git a/sdk/python/tests/test_verification.py b/sdk/python/tests/test_verification.py index 885266d2f..77841cc0d 100644 --- a/sdk/python/tests/test_verification.py +++ b/sdk/python/tests/test_verification.py @@ -1,6 +1,7 @@ """ Tests for agentfield.verification — LocalVerifier (security-critical). """ + from __future__ import annotations import time @@ -15,6 +16,7 @@ # Helpers # --------------------------------------------------------------------------- + def _make_verifier(**kwargs) -> LocalVerifier: return LocalVerifier(agentfield_url="http://localhost:8080", **kwargs) @@ -177,7 +179,9 @@ class TestVerifySignatureNoCrypto: def test_returns_false_without_cryptography(self): v = _make_verifier(timestamp_window=600) valid_ts = str(int(time.time())) - with patch.dict("sys.modules", {"cryptography.hazmat.primitives.asymmetric.ed25519": None}): + with patch.dict( + "sys.modules", {"cryptography.hazmat.primitives.asymmetric.ed25519": None} + ): result = v.verify_signature( caller_did="did:key:z6Mk", signature_b64="AAAA", @@ -391,9 +395,7 @@ def test_constraint_lte_violation_denies(self): "target_tags": [], "allow_functions": [], "deny_functions": [], - "constraints": { - "amount": {"operator": "<=", "value": 100} - }, + "constraints": {"amount": {"operator": "<=", "value": 100}}, } ] # amount=150 > 100 → constraint violated → deny @@ -410,9 +412,7 @@ def test_constraint_lte_satisfied_allows(self): "target_tags": [], "allow_functions": [], "deny_functions": [], - "constraints": { - "amount": {"operator": "<=", "value": 100} - }, + "constraints": {"amount": {"operator": "<=", "value": 100}}, } ] assert v.evaluate_policy([], [], "transfer", {"amount": 50}) is True diff --git a/sdk/python/tests/test_workflow_parent_child.py b/sdk/python/tests/test_workflow_parent_child.py index 4ba830311..47c507784 100644 --- a/sdk/python/tests/test_workflow_parent_child.py +++ b/sdk/python/tests/test_workflow_parent_child.py @@ -93,11 +93,13 @@ async def parent_reasoner(x: int, execution_context: ExecutionContext = None): # Extract execution IDs parent_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "parent_reasoner" and e["status"] == "running" ) child_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "child_reasoner" and e["status"] == "running" ) @@ -150,15 +152,18 @@ async def parent_reasoner(execution_context: ExecutionContext = None): assert len(captured_events) == 6 parent_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "parent_reasoner" and e["status"] == "running" ) child_b_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "child_b" and e["status"] == "running" ) child_c_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "child_c" and e["status"] == "running" ) @@ -206,15 +211,18 @@ async def parent(execution_context: ExecutionContext = None): # Get start events for each level parent_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "parent" and e["status"] == "running" ) child_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "child" and e["status"] == "running" ) grandchild_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "grandchild" and e["status"] == "running" ) @@ -314,23 +322,29 @@ async def test_decorated_reasoner_propagates_parent_context(self, monkeypatch): captured_payloads: List[Dict[str, Any]] = [] async def capture_start(agent, ctx, payload): - captured_payloads.append({ - "type": "start", - "execution_id": ctx.execution_id, - "parent_execution_id": ctx.parent_execution_id, - "reasoner_name": ctx.reasoner_name, - }) + captured_payloads.append( + { + "type": "start", + "execution_id": ctx.execution_id, + "parent_execution_id": ctx.parent_execution_id, + "reasoner_name": ctx.reasoner_name, + } + ) async def capture_complete(agent, ctx, result, duration_ms, payload): - captured_payloads.append({ - "type": "complete", - "execution_id": ctx.execution_id, - "parent_execution_id": ctx.parent_execution_id, - "reasoner_name": ctx.reasoner_name, - }) + captured_payloads.append( + { + "type": "complete", + "execution_id": ctx.execution_id, + "parent_execution_id": ctx.parent_execution_id, + "reasoner_name": ctx.reasoner_name, + } + ) monkeypatch.setattr("agentfield.decorators._send_workflow_start", capture_start) - monkeypatch.setattr("agentfield.decorators._send_workflow_completion", capture_complete) + monkeypatch.setattr( + "agentfield.decorators._send_workflow_completion", capture_complete + ) agent = StubAgent() set_current_agent(agent) @@ -366,7 +380,9 @@ async def parent_reasoner(x: int, execution_context: ExecutionContext = None): starts = [p for p in captured_payloads if p["type"] == "start"] assert len(starts) == 2 - parent_start = next(p for p in starts if p["reasoner_name"] == "parent_reasoner") + parent_start = next( + p for p in starts if p["reasoner_name"] == "parent_reasoner" + ) child_start = next(p for p in starts if p["reasoner_name"] == "child_reasoner") # Parent should have no parent @@ -410,11 +426,13 @@ async def parent_reasoner(execution_context: ExecutionContext = None): # Find parent start and child error events parent_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "parent_reasoner" and e["status"] == "running" ) child_error = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "failing_child" and e["status"] == "failed" ) @@ -461,17 +479,20 @@ async def parent_reasoner(execution_context: ExecutionContext = None): # Get parent start parent_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "parent_reasoner" and e["status"] == "running" ) # Both children should reference the same parent success_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "success_child" and e["status"] == "running" ) failing_start = next( - e for e in captured_events + e + for e in captured_events if e["reasoner_id"] == "failing_child" and e["status"] == "running" ) From f641dbedde2cd4e4369c03f27282c028f2315f2e Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Sat, 11 Jul 2026 05:57:26 +0000 Subject: [PATCH 6/9] fix: Fixed the `F401` lint error from `test_did_auth_invariants` test. --- sdk/python/tests/test_did_auth_invariants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/test_did_auth_invariants.py b/sdk/python/tests/test_did_auth_invariants.py index 124cebede..6d97a8088 100644 --- a/sdk/python/tests/test_did_auth_invariants.py +++ b/sdk/python/tests/test_did_auth_invariants.py @@ -152,8 +152,8 @@ def test_invariant_signature_is_deterministic_for_same_payload( """ try: from cryptography.hazmat.primitives.asymmetric.ed25519 import ( - Ed25519PrivateKey, - ) # noqa: F401 + Ed25519PrivateKey, # noqa: F401 + ) except ImportError: pytest.skip("cryptography library not available") From 5f1d116c125f910e553fe67cfbdb8bddcc0f8c2b Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Sat, 11 Jul 2026 06:22:41 +0000 Subject: [PATCH 7/9] test: Updated the test for updated `_cleanup_async_resources`. - Initially mock `Agent` is created through __new__ which by pass __init__ and it doesn't contain `_background_tasks` property. - I added `_background_tasks` property and added new assert in `test_cleanup_async_resources` test. --- sdk/python/tests/test_agent_core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/python/tests/test_agent_core.py b/sdk/python/tests/test_agent_core.py index 2eaa22384..f11be3a90 100644 --- a/sdk/python/tests/test_agent_core.py +++ b/sdk/python/tests/test_agent_core.py @@ -27,6 +27,7 @@ def make_agent_stub(): api_base="http://agentfield/api/v1", _get_auth_headers=lambda: {}, ) + agent._background_tasks = set() return agent @@ -79,6 +80,7 @@ async def stop(self): await agent._cleanup_async_resources() assert manager.stopped is True assert agent._async_execution_manager is None + assert len(agent._background_tasks) == 0 @pytest.mark.asyncio From 961e2dfc5d840e8828e9378b20e932f9a7227834 Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Mon, 13 Jul 2026 15:56:53 +0000 Subject: [PATCH 8/9] fix: Used a queue type notification dispatcher to maintain the order of notification. - class used a queue internally to maintain order. It itself run as background task so it will not block it's parent coroutine. - Initialization is done inside life cycle (lazily) so that queue and task attached them self with the ASGI optimized event loop (uvloop) - Some tests are modified as a new property introduced, so a similar dummy class also introduced for tests. --- sdk/python/agentfield/agent.py | 151 +++++++++++++++------ sdk/python/agentfield/agent_server.py | 3 + sdk/python/agentfield/agent_workflow.py | 55 +++++--- sdk/python/tests/helpers.py | 29 ++++ sdk/python/tests/test_agent_integration.py | 2 + sdk/python/tests/test_agent_server.py | 3 + 6 files changed, 181 insertions(+), 62 deletions(-) diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 57b2e1686..b155edda9 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -1,3 +1,4 @@ +from typing import Coroutine import asyncio import inspect import os @@ -500,6 +501,71 @@ def _bind_trigger_payload( return (), {} +class _NotificationDispatcher: + _SHUTDOWN = object() + + def __init__(self, dev_mode: bool): + self._queue: asyncio.Queue[Any] | None = None + self._dispatcher_task: asyncio.Task[Any] | None = None + self._dev_mode = dev_mode + + def start(self): + if self._dispatcher_task is not None: + return + self._queue = asyncio.Queue() + self._dispatcher_task = asyncio.create_task(self._run()) + + def is_start(self): + return self._dispatcher_task is not None + + def submit(self, coro_factory: Callable[[], Coroutine[Any, Any, None]]): + if self._queue is None: + if self._dev_mode: + log_error( + "Coroutine factory submitted before _NotoficationDispatcher even start" + ) + return + self._queue.put_nowait(coro_factory) + + async def _run(self): + if self._queue is None or self._dispatcher_task is None: + return + while True: + coro_factory = await self._queue.get() + if coro_factory is _NotificationDispatcher._SHUTDOWN: + self._queue.task_done() + break + try: + await coro_factory() + except Exception as e: + if self._dev_mode: + log_error(f"Notification dilivery failed: {e}") + finally: + self._queue.task_done() + + async def shutdown(self, timeout: int = 5): + if self._dispatcher_task is None or self._queue is None: + return + self._queue.put_nowait(_NotificationDispatcher._SHUTDOWN) + try: + await asyncio.wait_for(self._dispatcher_task, timeout=timeout) + except asyncio.TimeoutError: + self._dispatcher_task.cancel() + try: + await self._dispatcher_task + except asyncio.CancelledError: + pass + except Exception as e: + if self._dev_mode: + log_error(f"Notification dispatcher shutdown failed: {e}") + finally: + self._dispatcher_task = None + + +def _create_coro_factory(coro): + return lambda: coro + + class Agent(FastAPI): """ AgentField Agent - FastAPI subclass for creating AI agent nodes. @@ -712,6 +778,9 @@ def __init__( # prevent GC of fire-and-forget async execution tasks self._background_tasks: set[asyncio.Task] = set() + # Manage background notifications in order + self._notification_dispatcher = _NotificationDispatcher(dev_mode=self.dev_mode) + # Cooperative cancel registry. The control plane's cancel dispatcher # POSTs /_internal/executions/{id}/cancel to signal that the user's # reasoner code should stop. We track the active asyncio.Task per @@ -2265,8 +2334,9 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: execution_context.reasoner_name = reasoner_id - task = asyncio.create_task( - self.workflow_handler.notify_call_start( + + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_start( execution_context.execution_id, execution_context, reasoner_id, @@ -2274,8 +2344,6 @@ async def _execute_reasoner_endpoint( parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) start_time = time.time() @@ -2380,8 +2448,9 @@ async def _execute_reasoner_endpoint( if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - task = asyncio.create_task( - self.workflow_handler.notify_call_complete( + + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_complete( execution_context.execution_id, execution_context.workflow_id, result, @@ -2391,16 +2460,14 @@ async def _execute_reasoner_endpoint( parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) return result except asyncio.CancelledError as cancel_err: if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() - task = asyncio.create_task( - self.workflow_handler.notify_call_error( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_error( execution_context.execution_id, execution_context.workflow_id, "Execution cancelled by upstream client", @@ -2410,8 +2477,7 @@ async def _execute_reasoner_endpoint( parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + raise cancel_err except ExecuteError as exec_err: # Propagate upstream HTTP status codes from cross-agent calls. @@ -2419,20 +2485,20 @@ async def _execute_reasoner_endpoint( # (unhandled exception) and then 502 at the outer control plane. if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() + error_msg = str(exec_err) - task = asyncio.create_task( - self.workflow_handler.notify_call_error( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_error( execution_context.execution_id, execution_context.workflow_id, - str(exec_err), + error_msg, int((end_time - start_time) * 1000), execution_context, input_data=payload_dict, parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + detail = {"error": str(exec_err)} if exec_err.error_details: detail["error_details"] = exec_err.error_details @@ -2445,8 +2511,8 @@ async def _execute_reasoner_endpoint( end_time = time.time() detail = getattr(http_exc, "detail", None) or str(http_exc) - task = asyncio.create_task( - self.workflow_handler.notify_call_error( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_error( execution_context.execution_id, execution_context.workflow_id, detail, @@ -2456,26 +2522,25 @@ async def _execute_reasoner_endpoint( parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + raise except Exception as exc: if hasattr(self, "workflow_handler") and self.workflow_handler: end_time = time.time() + error_msg = str(exc) - task = asyncio.create_task( - self.workflow_handler.notify_call_error( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_error( execution_context.execution_id, execution_context.workflow_id, - str(exc), + error_msg, int((end_time - start_time) * 1000), execution_context, input_data=payload_dict, parent_execution_id=execution_context.parent_execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + raise finally: reset_execution_context(context_token) @@ -3161,8 +3226,8 @@ async def _run_async_skill(*args, **kwargs): self._current_execution_context = child_context input_payload = _build_invocation_payload(args, kwargs) - task = asyncio.create_task( - self.workflow_handler.notify_call_start( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_start( child_context.execution_id, child_context, skill_id, @@ -3170,16 +3235,14 @@ async def _run_async_skill(*args, **kwargs): parent_execution_id=current_context.execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) start_time = time.time() try: result = await original_func(*args, **kwargs) duration_ms = int((time.time() - start_time) * 1000) - task = asyncio.create_task( - self.workflow_handler.notify_call_complete( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_complete( child_context.execution_id, child_context.workflow_id, result, @@ -3189,25 +3252,24 @@ async def _run_async_skill(*args, **kwargs): parent_execution_id=current_context.execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + return result except Exception as exc: duration_ms = int((time.time() - start_time) * 1000) + error_msg = str(exc) - task = asyncio.create_task( - self.workflow_handler.notify_call_error( + self._notification_dispatcher.submit( + lambda: self.workflow_handler.notify_call_error( child_context.execution_id, child_context.workflow_id, - str(exc), + error_msg, duration_ms, child_context, input_data=input_payload, parent_execution_id=current_context.execution_id, ) ) - self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + raise finally: reset_execution_context(token) @@ -4448,9 +4510,18 @@ async def _cleanup_async_resources(self) -> None: log_debug("Background tasks are cleaned up") except Exception as e: if self.dev_mode: - log_debug(f"Error cleaning up background tasks: {e}") + log_error(f"Error cleaning up background tasks: {e}") finally: self._background_tasks.clear() + try: + await self._notification_dispatcher.shutdown() + if self.dev_mode: + log_debug( + "Notification dispatcher queue cleared and dispatcher shutdown" + ) + except Exception as e: + if self.dev_mode: + log_error(f"Error while shutdown notification dispatcher: {e}") if getattr(self, "client", None) is not None: try: diff --git a/sdk/python/agentfield/agent_server.py b/sdk/python/agentfield/agent_server.py index c438c377b..e4e4ed2e2 100644 --- a/sdk/python/agentfield/agent_server.py +++ b/sdk/python/agentfield/agent_server.py @@ -887,6 +887,9 @@ def on_disconnected(): # Start connection manager (non-blocking) connected = await self.agent.connection_manager.start() + # Start notification dispatcher + self.agent._notification_dispatcher.start() + # Connect memory event client - works independently of AgentField server connection if self.agent.memory_event_client: try: diff --git a/sdk/python/agentfield/agent_workflow.py b/sdk/python/agentfield/agent_workflow.py index ac6222132..2d9a45563 100644 --- a/sdk/python/agentfield/agent_workflow.py +++ b/sdk/python/agentfield/agent_workflow.py @@ -71,12 +71,14 @@ async def execute_with_tracking( start_time = time.time() parent_execution_id = parent_context.execution_id if parent_context else None - await self.notify_call_start( - execution_context.execution_id, - execution_context, - reasoner_name, - input_data, - parent_execution_id=parent_execution_id, + self.agent._notification_dispatcher.submit( + lambda: self.notify_call_start( + execution_context.execution_id, + execution_context, + reasoner_name, + input_data, + parent_execution_id=parent_execution_id, + ) ) try: @@ -84,27 +86,36 @@ async def execute_with_tracking( if inspect.isawaitable(result): result = await result duration_ms = int((time.time() - start_time) * 1000) - await self.notify_call_complete( - execution_context.execution_id, - execution_context.workflow_id, - result, - duration_ms, - execution_context, - input_data=input_data, - parent_execution_id=parent_execution_id, + + self.agent._notification_dispatcher.submit( + lambda: self.notify_call_complete( + execution_context.execution_id, + execution_context.workflow_id, + result, + duration_ms, + execution_context, + input_data=input_data, + parent_execution_id=parent_execution_id, + ) ) + return result except Exception as exc: # pragma: no cover - re-raised duration_ms = int((time.time() - start_time) * 1000) - await self.notify_call_error( - execution_context.execution_id, - execution_context.workflow_id, - str(exc), - duration_ms, - execution_context, - input_data=input_data, - parent_execution_id=parent_execution_id, + error_msg = str(exc) + + self.agent._notification_dispatcher.submit( + lambda: self.notify_call_error( + execution_context.execution_id, + execution_context.workflow_id, + error_msg, + duration_ms, + execution_context, + input_data=input_data, + parent_execution_id=parent_execution_id, + ) ) + raise finally: reset_execution_context(token) diff --git a/sdk/python/tests/helpers.py b/sdk/python/tests/helpers.py index e35dd8dd0..6f1b10184 100644 --- a/sdk/python/tests/helpers.py +++ b/sdk/python/tests/helpers.py @@ -1,6 +1,9 @@ """Shared testing utilities for AgentField SDK unit tests.""" from __future__ import annotations +from typing import Callable +from typing import Coroutine + import asyncio import threading @@ -92,6 +95,29 @@ def notify_graceful_shutdown_sync(self, node_id: str) -> bool: return True +class InlineTestingDispatcher: + def __init__(self): + self._started = True + + def start(self): + self._started = True + + def is_start(self): + return self._started + + def submit(self, coro_factory: Callable[[], Coroutine[Any, Any, None]]): + coro = coro_factory() + try: + coro.send(None) + except StopIteration: + pass + except RuntimeWarning: + pass + + async def shutdown(self, timeout: int = 5): + self._started = False + + @dataclass class StubAgent: """Light-weight stand-in for Agent used across module tests.""" @@ -113,6 +139,9 @@ class StubAgent: agentfield_connected: bool = True _current_status: AgentStatus = AgentStatus.STARTING callback_candidates: List[str] = field(default_factory=list) + _notification_dispatcher: InlineTestingDispatcher = field( + default_factory=InlineTestingDispatcher + ) def _build_vc_metadata(self): return {"agent_default": True} diff --git a/sdk/python/tests/test_agent_integration.py b/sdk/python/tests/test_agent_integration.py index 254305c0b..eb94ee17c 100644 --- a/sdk/python/tests/test_agent_integration.py +++ b/sdk/python/tests/test_agent_integration.py @@ -15,6 +15,8 @@ async def test_agent_reasoner_routing_and_workflow(monkeypatch): agent, agentfield_client = create_test_agent( monkeypatch, callback_url="https://callback.example.com" ) + # Manually start notification dispatcher + agent._notification_dispatcher.start() # Disable async execution for this test to get synchronous 200 responses agent.async_config.enable_async_execution = False # Disable agentfield_server to prevent async callback execution diff --git a/sdk/python/tests/test_agent_server.py b/sdk/python/tests/test_agent_server.py index 3ef708ee2..0ad262c83 100644 --- a/sdk/python/tests/test_agent_server.py +++ b/sdk/python/tests/test_agent_server.py @@ -43,6 +43,9 @@ def make_agent_app(**overrides): resolve_by_execution_id=AsyncMock(return_value=False), ), ) + app._notification_dispatcher = overrides.get( + "_notification_dispatcher", SimpleNamespace(start=MagicMock()) + ) return app From 424a551b5da6771d5285f6d68c61210ae99e1aa4 Mon Sep 17 00:00:00 2001 From: DebanKsahu Date: Mon, 13 Jul 2026 16:11:35 +0000 Subject: [PATCH 9/9] lint: Run ruff format --- examples/benchmarks/100k-scale/analyze.py | 136 ++++++---- .../100k-scale/crewai-bench/benchmark.py | 157 ++++++++--- .../100k-scale/langchain-bench/benchmark.py | 155 ++++++++--- .../100k-scale/python-bench/benchmark.py | 202 ++++++++++---- examples/e2e_resilience_tests/agent_flaky.py | 2 - .../e2e_resilience_tests/mock_llm_server.py | 78 ++++-- .../docker_hello_world/main.py | 1 - .../routers/ingestion.py | 8 +- .../documentation_chatbot/routers/qa.py | 4 +- .../python_agent_nodes/multi_version/main.py | 4 +- .../permission_agent_a/main.py | 4 +- .../python_agent_nodes/rag_evaluation/main.py | 1 + .../rag_evaluation/ml_services/embeddings.py | 9 +- .../rag_evaluation/ml_services/ner.py | 42 +-- .../rag_evaluation/ml_services/nli.py | 34 +-- .../rag_evaluation/models.py | 77 +++++- .../rag_evaluation/rag_eval_client.py | 255 +++++++++--------- .../reasoners/constitutional.py | 197 ++++++++------ .../rag_evaluation/reasoners/faithfulness.py | 179 +++++++----- .../rag_evaluation/reasoners/hallucination.py | 241 ++++++++++------- .../rag_evaluation/reasoners/orchestrator.py | 155 +++++------ .../rag_evaluation/reasoners/relevance.py | 123 +++++---- .../serverless_hello/main.py | 4 +- .../simulation_engine/example.ipynb | 14 +- .../simulation_engine/routers/aggregation.py | 6 +- .../simulation_engine/routers/decision.py | 2 +- .../simulation_engine/routers/scenario.py | 2 +- .../simulation_engine/routers/simulation.py | 2 +- .../python_agent_nodes/tool_calling/worker.py | 4 +- .../voice_dictation/main.py | 4 +- examples/triggers-demo/agent.py | 21 +- scripts/bump_version.py | 10 +- scripts/coverage-aggregate.py | 35 +-- scripts/coverage-gate.py | 185 +++++++------ scripts/tests/test_bump_version.py | 29 +- scripts/update_changelog.py | 4 +- tests/functional/agents/call_chain_agents.py | 6 +- .../agents/docs_quick_start_agent.py | 3 +- tests/functional/agents/memory_agent.py | 3 +- .../functional/agents/memory_events_agent.py | 24 +- .../agents/memory_events_decorator_agent.py | 7 +- tests/functional/agents/quick_start_agent.py | 7 +- .../functional/agents/restart_replay_agent.py | 7 +- .../functional/agents/router_prefix_agent.py | 3 +- tests/functional/agents/scoping_agent.py | 3 +- tests/functional/conftest.py | 80 +++--- tests/functional/tests/test_af_trigger_cli.py | 4 +- tests/functional/tests/test_ard_discovery.py | 33 ++- tests/functional/tests/test_go_sdk_cli.py | 114 ++++++-- tests/functional/tests/test_hello_world.py | 78 +++--- tests/functional/tests/test_memory_events.py | 32 +-- tests/functional/tests/test_quick_start.py | 25 +- tests/functional/tests/test_restart_replay.py | 17 +- .../functional/tests/test_scoping_headers.py | 19 +- .../tests/test_serverless_agents.py | 71 +++-- tests/functional/tests/test_ts_agent.py | 29 +- .../tests/test_ui_node_logs_proxy.py | 4 +- tests/functional/tests/test_vc_cli.py | 46 ++-- tests/functional/tests/test_waiting_state.py | 47 +++- tests/functional/utils/logging.py | 24 +- 60 files changed, 1921 insertions(+), 1151 deletions(-) diff --git a/examples/benchmarks/100k-scale/analyze.py b/examples/benchmarks/100k-scale/analyze.py index 8287eccff..b444b05a7 100644 --- a/examples/benchmarks/100k-scale/analyze.py +++ b/examples/benchmarks/100k-scale/analyze.py @@ -14,30 +14,32 @@ import numpy as np # Clean, minimal styling -plt.rcParams.update({ - "figure.dpi": 150, - "savefig.dpi": 300, - "font.family": "sans-serif", - "font.size": 11, - "axes.labelsize": 12, - "axes.titlesize": 13, - "axes.spines.top": False, - "axes.spines.right": False, - "axes.grid": True, - "grid.alpha": 0.2, - "grid.linewidth": 0.5, -}) +plt.rcParams.update( + { + "figure.dpi": 150, + "savefig.dpi": 300, + "font.family": "sans-serif", + "font.size": 11, + "axes.labelsize": 12, + "axes.titlesize": 13, + "axes.spines.top": False, + "axes.spines.right": False, + "axes.grid": True, + "grid.alpha": 0.2, + "grid.linewidth": 0.5, + } +) # Color palette: AgentField SDKs in blue gradient, external frameworks in distinct colors COLORS = { # AgentField SDKs - Blue family (visually grouped) - "AgentField_Go": "#0D2137", # Deep navy - "AgentField_TypeScript": "#1A5276", # Medium blue - "AgentField_Python": "#3498DB", # Bright blue + "AgentField_Go": "#0D2137", # Deep navy + "AgentField_TypeScript": "#1A5276", # Medium blue + "AgentField_Python": "#3498DB", # Bright blue # External frameworks - "LangChain_Python": "#C0392B", # Red - "CrewAI_Python": "#27AE60", # Green - "Mastra_TypeScript": "#E67E22", # Orange + "LangChain_Python": "#C0392B", # Red + "CrewAI_Python": "#27AE60", # Green + "Mastra_TypeScript": "#E67E22", # Orange } LABELS = { @@ -77,7 +79,9 @@ def create_benchmark_figure(results: dict, output_dir: Path): Create a single clean figure with 4 key metrics. """ fig, axes = plt.subplots(2, 2, figsize=(14, 10)) - fig.suptitle("Agent Framework Benchmark Comparison", fontsize=16, fontweight="bold", y=0.95) + fig.suptitle( + "Agent Framework Benchmark Comparison", fontsize=16, fontweight="bold", y=0.95 + ) # Framework order: AgentField SDKs first (grouped), then external frameworks = [ @@ -96,18 +100,24 @@ def plot_metric(ax, metric_name, alt_metric, title, unit): colors = [] for fw in frameworks: - v = get_metric(results, fw, metric_name) or get_metric(results, fw, alt_metric) + v = get_metric(results, fw, metric_name) or get_metric( + results, fw, alt_metric + ) if v is not None: values.append(v) labels.append(LABELS[fw]) colors.append(COLORS[fw]) if not values: - ax.text(0.5, 0.5, "No data", ha="center", va="center", transform=ax.transAxes) + ax.text( + 0.5, 0.5, "No data", ha="center", va="center", transform=ax.transAxes + ) return y_pos = np.arange(len(labels)) - bars = ax.barh(y_pos, values, color=colors, edgecolor="white", linewidth=1, height=0.65) + bars = ax.barh( + y_pos, values, color=colors, edgecolor="white", linewidth=1, height=0.65 + ) ax.set_yticks(y_pos) ax.set_yticklabels(labels) @@ -122,45 +132,77 @@ def plot_metric(ax, metric_name, alt_metric, title, unit): # Add value labels for bar, val in zip(bars, values): if val >= 1_000_000: - label = f"{val/1_000_000:.1f}M" + label = f"{val / 1_000_000:.1f}M" elif val >= 1_000: - label = f"{val/1_000:.1f}K" + label = f"{val / 1_000:.1f}K" elif val >= 1: label = f"{val:.1f}" else: label = f"{val:.2f}" x_pos = bar.get_width() - ax.text(x_pos * 1.08, bar.get_y() + bar.get_height()/2, - label, va="center", fontsize=10, fontweight="bold") + ax.text( + x_pos * 1.08, + bar.get_y() + bar.get_height() / 2, + label, + va="center", + fontsize=10, + fontweight="bold", + ) # Plot 4 key metrics - plot_metric(axes[0, 0], - "registration_time_mean_ms", "registration_time_mean_ms", - "Registration Time (1000 handlers)", "milliseconds") - - plot_metric(axes[0, 1], - "memory_per_handler_bytes", "memory_per_tool_bytes", - "Memory per Handler", "bytes") - - plot_metric(axes[1, 0], - "request_latency_p99_us", "invocation_latency_p99_us", - "Invocation Latency (p99)", "microseconds") - - plot_metric(axes[1, 1], - "theoretical_single_thread_rps", "theoretical_single_thread_rps", - "Throughput", "requests/second") + plot_metric( + axes[0, 0], + "registration_time_mean_ms", + "registration_time_mean_ms", + "Registration Time (1000 handlers)", + "milliseconds", + ) + + plot_metric( + axes[0, 1], + "memory_per_handler_bytes", + "memory_per_tool_bytes", + "Memory per Handler", + "bytes", + ) + + plot_metric( + axes[1, 0], + "request_latency_p99_us", + "invocation_latency_p99_us", + "Invocation Latency (p99)", + "microseconds", + ) + + plot_metric( + axes[1, 1], + "theoretical_single_thread_rps", + "theoretical_single_thread_rps", + "Throughput", + "requests/second", + ) plt.tight_layout(rect=[0, 0.06, 1, 0.93]) # Add legend with visual grouping present = [fw for fw in frameworks if fw in results] - handles = [plt.Rectangle((0,0), 1, 1, color=COLORS[fw]) for fw in present] + handles = [plt.Rectangle((0, 0), 1, 1, color=COLORS[fw]) for fw in present] legend_labels = [LABELS[fw] for fw in present] - fig.legend(handles, legend_labels, loc="lower center", ncol=min(len(present), 6), - frameon=True, framealpha=0.95, edgecolor="0.8", fontsize=10) - - fig.savefig(output_dir / "benchmark_summary.png", bbox_inches="tight", facecolor="white") + fig.legend( + handles, + legend_labels, + loc="lower center", + ncol=min(len(present), 6), + frameon=True, + framealpha=0.95, + edgecolor="0.8", + fontsize=10, + ) + + fig.savefig( + output_dir / "benchmark_summary.png", bbox_inches="tight", facecolor="white" + ) plt.close() print(f"Saved: {output_dir / 'benchmark_summary.png'}") diff --git a/examples/benchmarks/100k-scale/crewai-bench/benchmark.py b/examples/benchmarks/100k-scale/crewai-bench/benchmark.py index ccc6e00f0..8dbd60418 100644 --- a/examples/benchmarks/100k-scale/crewai-bench/benchmark.py +++ b/examples/benchmarks/100k-scale/crewai-bench/benchmark.py @@ -19,6 +19,7 @@ try: from crewai.tools import BaseTool, tool from pydantic import BaseModel, Field + CREWAI_AVAILABLE = True except ImportError: CREWAI_AVAILABLE = False @@ -60,7 +61,9 @@ def percentile(p: float) -> float: ) -def benchmark_tool_registration(num_tools: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_tool_registration( + num_tools: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure time to create CrewAI tools using BaseTool subclassing.""" if not CREWAI_AVAILABLE: return [] @@ -94,7 +97,7 @@ class ToolInput(BaseModel): "description": f"Tool number {idx}", "args_schema": ToolInput, "_run": lambda self, query: {"tool_id": idx, "processed": True}, - } + }, ) tools.append(tool_class()) @@ -111,7 +114,9 @@ class ToolInput(BaseModel): return results -def benchmark_tool_decorator(num_tools: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_tool_decorator( + num_tools: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure time to create CrewAI tools using @tool decorator.""" if not CREWAI_AVAILABLE: return [] @@ -137,6 +142,7 @@ def make_tool(tool_idx): def tool_func(query: str) -> dict: """Process a query and return result.""" return {"tool_id": tool_idx, "processed": True} + return tool_func tools.append(make_tool(idx)) @@ -154,7 +160,9 @@ def tool_func(query: str) -> dict: return results -def benchmark_memory(num_tools: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_memory( + num_tools: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure memory footprint of CrewAI tools.""" if not CREWAI_AVAILABLE: return [] @@ -180,6 +188,7 @@ def make_tool(tool_idx): def tool_func(query: str) -> dict: """Process a query.""" return {"tool_id": tool_idx} + return tool_func tools.append(make_tool(idx)) @@ -233,7 +242,9 @@ def ping_tool(query: str) -> dict: return results -def benchmark_tool_invocation(num_tools: int, num_invocations: int, verbose: bool) -> list[float]: +def benchmark_tool_invocation( + num_tools: int, num_invocations: int, verbose: bool +) -> list[float]: """Measure tool invocation latency.""" if not CREWAI_AVAILABLE: return [] @@ -255,6 +266,7 @@ def tool_func(query: str) -> dict: "processed": True, "timestamp": time.time_ns(), } + return tool_func tools.append(make_tool(idx)) @@ -274,7 +286,9 @@ def tool_func(query: str) -> dict: if verbose: stats = calculate_stats(results) - print(f" p50: {stats.p50:.2f} us, p95: {stats.p95:.2f} us, p99: {stats.p99:.2f} us") + print( + f" p50: {stats.p50:.2f} us, p95: {stats.p95:.2f} us, p99: {stats.p99:.2f} us" + ) return results @@ -282,7 +296,9 @@ def tool_func(query: str) -> dict: def main(): parser = argparse.ArgumentParser(description="CrewAI Benchmark") parser.add_argument("--tools", type=int, default=1000, help="Number of tools") - parser.add_argument("--iterations", type=int, default=10, help="Benchmark iterations") + parser.add_argument( + "--iterations", type=int, default=10, help="Benchmark iterations" + ) parser.add_argument("--warmup", type=int, default=2, help="Warmup iterations") parser.add_argument("--json", action="store_true", help="JSON output") args = parser.parse_args() @@ -309,20 +325,44 @@ def main(): if verbose: print("CrewAI Benchmark") print("================") - print(f"Tools: {args.tools} | Iterations: {args.iterations} | Warmup: {args.warmup}\n") + print( + f"Tools: {args.tools} | Iterations: {args.iterations} | Warmup: {args.warmup}\n" + ) # Registration benchmark (using @tool decorator - more common usage) reg_tools = min(args.tools, 1000) - reg_times = benchmark_tool_decorator(reg_tools, args.iterations, args.warmup, verbose) + reg_times = benchmark_tool_decorator( + reg_tools, args.iterations, args.warmup, verbose + ) if reg_times: reg_stats = calculate_stats(reg_times) suite["raw_data"]["registration_time_ms"] = reg_times - suite["results"].extend([ - {"metric": "registration_time_mean_ms", "value": reg_stats.mean, "unit": "ms", "iterations": len(reg_times), "tool_count": reg_tools}, - {"metric": "registration_time_stddev_ms", "value": reg_stats.stddev, "unit": "ms"}, - {"metric": "registration_time_p50_ms", "value": reg_stats.p50, "unit": "ms"}, - {"metric": "registration_time_p99_ms", "value": reg_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "registration_time_mean_ms", + "value": reg_stats.mean, + "unit": "ms", + "iterations": len(reg_times), + "tool_count": reg_tools, + }, + { + "metric": "registration_time_stddev_ms", + "value": reg_stats.stddev, + "unit": "ms", + }, + { + "metric": "registration_time_p50_ms", + "value": reg_stats.p50, + "unit": "ms", + }, + { + "metric": "registration_time_p99_ms", + "value": reg_stats.p99, + "unit": "ms", + }, + ] + ) # Memory benchmark mem_tools = min(args.tools, 1000) @@ -330,49 +370,92 @@ def main(): if mem_data: mem_stats = calculate_stats(mem_data) suite["raw_data"]["memory_mb"] = mem_data - suite["results"].extend([ - {"metric": "memory_mean_mb", "value": mem_stats.mean, "unit": "MB", "iterations": len(mem_data), "tool_count": mem_tools}, - {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, - {"metric": "memory_per_tool_bytes", "value": (mem_stats.mean * 1024 * 1024) / mem_tools, "unit": "bytes"}, - ]) + suite["results"].extend( + [ + { + "metric": "memory_mean_mb", + "value": mem_stats.mean, + "unit": "MB", + "iterations": len(mem_data), + "tool_count": mem_tools, + }, + {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, + { + "metric": "memory_per_tool_bytes", + "value": (mem_stats.mean * 1024 * 1024) / mem_tools, + "unit": "bytes", + }, + ] + ) # Cold start benchmark cold_times = benchmark_cold_start(args.iterations, args.warmup, verbose) if cold_times: cold_stats = calculate_stats(cold_times) suite["raw_data"]["cold_start_ms"] = cold_times - suite["results"].extend([ - {"metric": "cold_start_mean_ms", "value": cold_stats.mean, "unit": "ms", "iterations": len(cold_times)}, - {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "cold_start_mean_ms", + "value": cold_stats.mean, + "unit": "ms", + "iterations": len(cold_times), + }, + {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, + ] + ) # Invocation latency benchmark inv_times = benchmark_tool_invocation(min(args.tools, 100), 10000, verbose) if inv_times: inv_stats = calculate_stats(inv_times) suite["raw_data"]["invocation_latency_us"] = inv_times - suite["results"].extend([ - {"metric": "invocation_latency_mean_us", "value": inv_stats.mean, "unit": "us"}, - {"metric": "invocation_latency_p50_us", "value": inv_stats.p50, "unit": "us"}, - {"metric": "invocation_latency_p95_us", "value": inv_stats.p95, "unit": "us"}, - {"metric": "invocation_latency_p99_us", "value": inv_stats.p99, "unit": "us"}, - ]) + suite["results"].extend( + [ + { + "metric": "invocation_latency_mean_us", + "value": inv_stats.mean, + "unit": "us", + }, + { + "metric": "invocation_latency_p50_us", + "value": inv_stats.p50, + "unit": "us", + }, + { + "metric": "invocation_latency_p95_us", + "value": inv_stats.p95, + "unit": "us", + }, + { + "metric": "invocation_latency_p99_us", + "value": inv_stats.p99, + "unit": "us", + }, + ] + ) if inv_stats.mean > 0: - suite["results"].append({ - "metric": "theoretical_single_thread_rps", - "value": 1_000_000 / inv_stats.mean, - "unit": "req/s", - }) + suite["results"].append( + { + "metric": "theoretical_single_thread_rps", + "value": 1_000_000 / inv_stats.mean, + "unit": "req/s", + } + ) if args.json: print(json.dumps(suite, indent=2)) else: print("\n=== Summary ===") if reg_times: - print(f"Registration ({reg_tools}): {reg_stats.mean:.2f} ms (+/-{reg_stats.stddev:.2f})") + print( + f"Registration ({reg_tools}): {reg_stats.mean:.2f} ms (+/-{reg_stats.stddev:.2f})" + ) if mem_data: - print(f"Memory ({mem_tools}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_tools:.0f} bytes/tool)") + print( + f"Memory ({mem_tools}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_tools:.0f} bytes/tool)" + ) if cold_times: print(f"Cold Start: {cold_stats.mean:.2f} ms") if inv_times: diff --git a/examples/benchmarks/100k-scale/langchain-bench/benchmark.py b/examples/benchmarks/100k-scale/langchain-bench/benchmark.py index 5b94873ad..4321c473b 100644 --- a/examples/benchmarks/100k-scale/langchain-bench/benchmark.py +++ b/examples/benchmarks/100k-scale/langchain-bench/benchmark.py @@ -19,10 +19,13 @@ try: from langchain_core.tools import tool, StructuredTool + LANGCHAIN_AVAILABLE = True except ImportError: LANGCHAIN_AVAILABLE = False - print("Warning: langchain-core not installed. Install with: pip install langchain-core") + print( + "Warning: langchain-core not installed. Install with: pip install langchain-core" + ) @dataclass @@ -60,7 +63,9 @@ def percentile(p: float) -> float: ) -def benchmark_tool_registration(num_tools: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_tool_registration( + num_tools: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure time to create LangChain tools.""" if not LANGCHAIN_AVAILABLE: return [] @@ -84,6 +89,7 @@ def benchmark_tool_registration(num_tools: int, iterations: int, warmup: int, ve def make_func(tool_idx): def tool_func(query: str) -> dict: return {"tool_id": tool_idx, "processed": True} + return tool_func t = StructuredTool.from_function( @@ -106,7 +112,9 @@ def tool_func(query: str) -> dict: return results -def benchmark_memory(num_tools: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_memory( + num_tools: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure memory footprint of LangChain tools.""" if not LANGCHAIN_AVAILABLE: return [] @@ -129,6 +137,7 @@ def benchmark_memory(num_tools: int, iterations: int, warmup: int, verbose: bool def make_func(tool_idx): def tool_func(query: str) -> dict: return {"tool_id": tool_idx} + return tool_func t = StructuredTool.from_function( @@ -188,7 +197,9 @@ def ping(query: str) -> dict: return results -def benchmark_tool_invocation(num_tools: int, num_invocations: int, verbose: bool) -> list[float]: +def benchmark_tool_invocation( + num_tools: int, num_invocations: int, verbose: bool +) -> list[float]: """Measure tool invocation latency.""" if not LANGCHAIN_AVAILABLE: return [] @@ -208,6 +219,7 @@ def tool_func(query: str) -> dict: "processed": True, "timestamp": time.time_ns(), } + return tool_func t = StructuredTool.from_function( @@ -232,7 +244,9 @@ def tool_func(query: str) -> dict: if verbose: stats = calculate_stats(results) - print(f" p50: {stats.p50:.2f} µs, p95: {stats.p95:.2f} µs, p99: {stats.p99:.2f} µs") + print( + f" p50: {stats.p50:.2f} µs, p95: {stats.p95:.2f} µs, p99: {stats.p99:.2f} µs" + ) return results @@ -240,7 +254,9 @@ def tool_func(query: str) -> dict: def main(): parser = argparse.ArgumentParser(description="LangChain Baseline Benchmark") parser.add_argument("--tools", type=int, default=1000, help="Number of tools") - parser.add_argument("--iterations", type=int, default=10, help="Benchmark iterations") + parser.add_argument( + "--iterations", type=int, default=10, help="Benchmark iterations" + ) parser.add_argument("--warmup", type=int, default=2, help="Warmup iterations") parser.add_argument("--json", action="store_true", help="JSON output") args = parser.parse_args() @@ -267,20 +283,44 @@ def main(): if verbose: print("LangChain Baseline Benchmark") print("============================") - print(f"Tools: {args.tools} | Iterations: {args.iterations} | Warmup: {args.warmup}\n") + print( + f"Tools: {args.tools} | Iterations: {args.iterations} | Warmup: {args.warmup}\n" + ) # Registration benchmark reg_tools = min(args.tools, 1000) # Limit to 1000 for reasonable benchmark time - reg_times = benchmark_tool_registration(reg_tools, args.iterations, args.warmup, verbose) + reg_times = benchmark_tool_registration( + reg_tools, args.iterations, args.warmup, verbose + ) if reg_times: reg_stats = calculate_stats(reg_times) suite["raw_data"]["registration_time_ms"] = reg_times - suite["results"].extend([ - {"metric": "registration_time_mean_ms", "value": reg_stats.mean, "unit": "ms", "iterations": len(reg_times), "tool_count": reg_tools}, - {"metric": "registration_time_stddev_ms", "value": reg_stats.stddev, "unit": "ms"}, - {"metric": "registration_time_p50_ms", "value": reg_stats.p50, "unit": "ms"}, - {"metric": "registration_time_p99_ms", "value": reg_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "registration_time_mean_ms", + "value": reg_stats.mean, + "unit": "ms", + "iterations": len(reg_times), + "tool_count": reg_tools, + }, + { + "metric": "registration_time_stddev_ms", + "value": reg_stats.stddev, + "unit": "ms", + }, + { + "metric": "registration_time_p50_ms", + "value": reg_stats.p50, + "unit": "ms", + }, + { + "metric": "registration_time_p99_ms", + "value": reg_stats.p99, + "unit": "ms", + }, + ] + ) # Memory benchmark mem_tools = min(args.tools, 1000) @@ -288,49 +328,92 @@ def main(): if mem_data: mem_stats = calculate_stats(mem_data) suite["raw_data"]["memory_mb"] = mem_data - suite["results"].extend([ - {"metric": "memory_mean_mb", "value": mem_stats.mean, "unit": "MB", "iterations": len(mem_data), "tool_count": mem_tools}, - {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, - {"metric": "memory_per_tool_bytes", "value": (mem_stats.mean * 1024 * 1024) / mem_tools, "unit": "bytes"}, - ]) + suite["results"].extend( + [ + { + "metric": "memory_mean_mb", + "value": mem_stats.mean, + "unit": "MB", + "iterations": len(mem_data), + "tool_count": mem_tools, + }, + {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, + { + "metric": "memory_per_tool_bytes", + "value": (mem_stats.mean * 1024 * 1024) / mem_tools, + "unit": "bytes", + }, + ] + ) # Cold start benchmark cold_times = benchmark_cold_start(args.iterations, args.warmup, verbose) if cold_times: cold_stats = calculate_stats(cold_times) suite["raw_data"]["cold_start_ms"] = cold_times - suite["results"].extend([ - {"metric": "cold_start_mean_ms", "value": cold_stats.mean, "unit": "ms", "iterations": len(cold_times)}, - {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "cold_start_mean_ms", + "value": cold_stats.mean, + "unit": "ms", + "iterations": len(cold_times), + }, + {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, + ] + ) # Invocation latency benchmark inv_times = benchmark_tool_invocation(min(args.tools, 100), 10000, verbose) if inv_times: inv_stats = calculate_stats(inv_times) suite["raw_data"]["invocation_latency_us"] = inv_times - suite["results"].extend([ - {"metric": "invocation_latency_mean_us", "value": inv_stats.mean, "unit": "us"}, - {"metric": "invocation_latency_p50_us", "value": inv_stats.p50, "unit": "us"}, - {"metric": "invocation_latency_p95_us", "value": inv_stats.p95, "unit": "us"}, - {"metric": "invocation_latency_p99_us", "value": inv_stats.p99, "unit": "us"}, - ]) + suite["results"].extend( + [ + { + "metric": "invocation_latency_mean_us", + "value": inv_stats.mean, + "unit": "us", + }, + { + "metric": "invocation_latency_p50_us", + "value": inv_stats.p50, + "unit": "us", + }, + { + "metric": "invocation_latency_p95_us", + "value": inv_stats.p95, + "unit": "us", + }, + { + "metric": "invocation_latency_p99_us", + "value": inv_stats.p99, + "unit": "us", + }, + ] + ) if inv_stats.mean > 0: - suite["results"].append({ - "metric": "theoretical_single_thread_rps", - "value": 1_000_000 / inv_stats.mean, - "unit": "req/s", - }) + suite["results"].append( + { + "metric": "theoretical_single_thread_rps", + "value": 1_000_000 / inv_stats.mean, + "unit": "req/s", + } + ) if args.json: print(json.dumps(suite, indent=2)) else: print("\n=== Summary ===") if reg_times: - print(f"Registration ({reg_tools}): {reg_stats.mean:.2f} ms (±{reg_stats.stddev:.2f})") + print( + f"Registration ({reg_tools}): {reg_stats.mean:.2f} ms (±{reg_stats.stddev:.2f})" + ) if mem_data: - print(f"Memory ({mem_tools}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_tools:.0f} bytes/tool)") + print( + f"Memory ({mem_tools}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_tools:.0f} bytes/tool)" + ) if cold_times: print(f"Cold Start: {cold_stats.mean:.2f} ms") if inv_times: diff --git a/examples/benchmarks/100k-scale/python-bench/benchmark.py b/examples/benchmarks/100k-scale/python-bench/benchmark.py index 483e6fcdb..a31c01aa3 100644 --- a/examples/benchmarks/100k-scale/python-bench/benchmark.py +++ b/examples/benchmarks/100k-scale/python-bench/benchmark.py @@ -22,7 +22,9 @@ from typing import Any # Add SDK to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'sdk', 'python')) +sys.path.insert( + 0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "sdk", "python") +) from agentfield import Agent @@ -79,7 +81,6 @@ def benchmark_agent_init(iterations: int, warmup: int, verbose: bool) -> list[fl node_id=f"init-bench-{i}", agentfield_server="http://localhost:8080", auto_register=False, - ) elapsed_ms = (time.perf_counter() - start) * 1000 @@ -95,7 +96,9 @@ def benchmark_agent_init(iterations: int, warmup: int, verbose: bool) -> list[fl return results -def benchmark_handler_registration(num_handlers: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_handler_registration( + num_handlers: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure ONLY handler registration time (Agent already created).""" if verbose: print(f"\nBenchmark: Handler Registration ONLY ({num_handlers} handlers)") @@ -111,7 +114,6 @@ def benchmark_handler_registration(num_handlers: int, iterations: int, warmup: i node_id=f"handler-bench-{i}", agentfield_server="http://localhost:8080", auto_register=False, - ) # Measure ONLY handler registration @@ -130,7 +132,9 @@ async def handler(input_data: dict, _idx=idx) -> dict: results.append(elapsed_ms) if verbose: per_handler_ms = elapsed_ms / num_handlers - print(f" Run {i - warmup + 1}: {elapsed_ms:.2f} ms ({per_handler_ms:.3f} ms/handler)") + print( + f" Run {i - warmup + 1}: {elapsed_ms:.2f} ms ({per_handler_ms:.3f} ms/handler)" + ) del agent gc.collect() @@ -156,7 +160,6 @@ def benchmark_agent_memory(iterations: int, warmup: int, verbose: bool) -> list[ node_id=f"agent-mem-{i}", agentfield_server="http://localhost:8080", auto_register=False, - ) gc.collect() @@ -176,7 +179,9 @@ def benchmark_agent_memory(iterations: int, warmup: int, verbose: bool) -> list[ return results -def benchmark_handler_memory(num_handlers: int, iterations: int, warmup: int, verbose: bool) -> list[float]: +def benchmark_handler_memory( + num_handlers: int, iterations: int, warmup: int, verbose: bool +) -> list[float]: """Measure memory for handlers ONLY (Agent overhead excluded).""" if verbose: print(f"\nBenchmark: Handler Memory ONLY ({num_handlers} handlers)") @@ -193,7 +198,6 @@ def benchmark_handler_memory(num_handlers: int, iterations: int, warmup: int, ve node_id=f"handler-mem-{i}", agentfield_server="http://localhost:8080", auto_register=False, - ) gc.collect() @@ -220,7 +224,9 @@ async def handler(input_data: dict, _idx=idx) -> dict: results.append(mem_mb) if verbose: per_handler_kb = (mem_mb * 1024) / num_handlers - print(f" Run {i - warmup + 1}: {mem_mb:.2f} MB ({per_handler_kb:.2f} KB/handler)") + print( + f" Run {i - warmup + 1}: {mem_mb:.2f} MB ({per_handler_kb:.2f} KB/handler)" + ) del agent gc.collect() @@ -244,7 +250,6 @@ def benchmark_cold_start(iterations: int, warmup: int, verbose: bool) -> list[fl node_id=f"cold-{i}", agentfield_server="http://localhost:8080", auto_register=False, - ) @agent.reasoner("ping") @@ -263,7 +268,9 @@ async def ping(input_data: dict) -> dict: return results -async def benchmark_request_processing(num_handlers: int, num_requests: int, verbose: bool) -> list[float]: +async def benchmark_request_processing( + num_handlers: int, num_requests: int, verbose: bool +) -> list[float]: """Measure request processing latency (handler invocation only).""" if verbose: print(f"\nBenchmark: Request Processing Latency ({num_requests} requests)") @@ -298,15 +305,21 @@ async def handler(input_data: dict, _idx=idx) -> dict: if verbose: stats = calculate_stats(results) - print(f" p50: {stats.p50:.2f} µs, p95: {stats.p95:.2f} µs, p99: {stats.p99:.2f} µs") + print( + f" p50: {stats.p50:.2f} µs, p95: {stats.p95:.2f} µs, p99: {stats.p99:.2f} µs" + ) return results def main(): parser = argparse.ArgumentParser(description="AgentField Python SDK Benchmark") - parser.add_argument("--handlers", type=int, default=10000, help="Number of handlers") - parser.add_argument("--iterations", type=int, default=10, help="Benchmark iterations") + parser.add_argument( + "--handlers", type=int, default=10000, help="Number of handlers" + ) + parser.add_argument( + "--iterations", type=int, default=10, help="Benchmark iterations" + ) parser.add_argument("--warmup", type=int, default=2, help="Warmup iterations") parser.add_argument("--json", action="store_true", help="JSON output") args = parser.parse_args() @@ -329,76 +342,153 @@ def main(): if verbose: print("AgentField Python SDK Benchmark (Fixed Methodology)") print("====================================================") - print(f"Handlers: {args.handlers} | Iterations: {args.iterations} | Warmup: {args.warmup}") + print( + f"Handlers: {args.handlers} | Iterations: {args.iterations} | Warmup: {args.warmup}" + ) print("\nNOTE: Agent init and handler registration are measured SEPARATELY\n") # 1. Agent initialization time (no handlers) agent_init_times = benchmark_agent_init(args.iterations, args.warmup, verbose) agent_init_stats = calculate_stats(agent_init_times) suite["raw_data"]["agent_init_time_ms"] = agent_init_times - suite["results"].extend([ - {"metric": "agent_init_time_mean_ms", "value": agent_init_stats.mean, "unit": "ms", "iterations": len(agent_init_times)}, - {"metric": "agent_init_time_p99_ms", "value": agent_init_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "agent_init_time_mean_ms", + "value": agent_init_stats.mean, + "unit": "ms", + "iterations": len(agent_init_times), + }, + { + "metric": "agent_init_time_p99_ms", + "value": agent_init_stats.p99, + "unit": "ms", + }, + ] + ) # 2. Handler registration time ONLY (Agent created outside measurement) reg_handlers = min(args.handlers, 10000) # Can test up to 10K now - reg_times = benchmark_handler_registration(reg_handlers, args.iterations, args.warmup, verbose) + reg_times = benchmark_handler_registration( + reg_handlers, args.iterations, args.warmup, verbose + ) reg_stats = calculate_stats(reg_times) suite["raw_data"]["registration_time_ms"] = reg_times - suite["results"].extend([ - {"metric": "registration_time_mean_ms", "value": reg_stats.mean, "unit": "ms", "iterations": len(reg_times), "handler_count": reg_handlers}, - {"metric": "registration_time_stddev_ms", "value": reg_stats.stddev, "unit": "ms"}, - {"metric": "registration_time_p50_ms", "value": reg_stats.p50, "unit": "ms"}, - {"metric": "registration_time_p99_ms", "value": reg_stats.p99, "unit": "ms"}, - {"metric": "registration_time_per_handler_ms", "value": reg_stats.mean / reg_handlers, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "registration_time_mean_ms", + "value": reg_stats.mean, + "unit": "ms", + "iterations": len(reg_times), + "handler_count": reg_handlers, + }, + { + "metric": "registration_time_stddev_ms", + "value": reg_stats.stddev, + "unit": "ms", + }, + { + "metric": "registration_time_p50_ms", + "value": reg_stats.p50, + "unit": "ms", + }, + { + "metric": "registration_time_p99_ms", + "value": reg_stats.p99, + "unit": "ms", + }, + { + "metric": "registration_time_per_handler_ms", + "value": reg_stats.mean / reg_handlers, + "unit": "ms", + }, + ] + ) # 3. Agent memory (no handlers) agent_mem_data = benchmark_agent_memory(args.iterations, args.warmup, verbose) agent_mem_stats = calculate_stats(agent_mem_data) suite["raw_data"]["agent_memory_mb"] = agent_mem_data - suite["results"].extend([ - {"metric": "agent_memory_mean_mb", "value": agent_mem_stats.mean, "unit": "MB", "iterations": len(agent_mem_data)}, - ]) + suite["results"].extend( + [ + { + "metric": "agent_memory_mean_mb", + "value": agent_mem_stats.mean, + "unit": "MB", + "iterations": len(agent_mem_data), + }, + ] + ) # 4. Handler memory ONLY (Agent excluded) mem_handlers = min(args.handlers, 10000) - mem_data = benchmark_handler_memory(mem_handlers, args.iterations, args.warmup, verbose) + mem_data = benchmark_handler_memory( + mem_handlers, args.iterations, args.warmup, verbose + ) mem_stats = calculate_stats(mem_data) suite["raw_data"]["memory_mb"] = mem_data - suite["results"].extend([ - {"metric": "memory_mean_mb", "value": mem_stats.mean, "unit": "MB", "iterations": len(mem_data), "handler_count": mem_handlers}, - {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, - {"metric": "memory_per_handler_bytes", "value": (mem_stats.mean * 1024 * 1024) / mem_handlers, "unit": "bytes"}, - ]) + suite["results"].extend( + [ + { + "metric": "memory_mean_mb", + "value": mem_stats.mean, + "unit": "MB", + "iterations": len(mem_data), + "handler_count": mem_handlers, + }, + {"metric": "memory_stddev_mb", "value": mem_stats.stddev, "unit": "MB"}, + { + "metric": "memory_per_handler_bytes", + "value": (mem_stats.mean * 1024 * 1024) / mem_handlers, + "unit": "bytes", + }, + ] + ) # 5. Cold start (Agent + 1 handler, for comparison with other benchmarks) cold_times = benchmark_cold_start(args.iterations, args.warmup, verbose) cold_stats = calculate_stats(cold_times) suite["raw_data"]["cold_start_ms"] = cold_times - suite["results"].extend([ - {"metric": "cold_start_mean_ms", "value": cold_stats.mean, "unit": "ms", "iterations": len(cold_times)}, - {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, - ]) + suite["results"].extend( + [ + { + "metric": "cold_start_mean_ms", + "value": cold_stats.mean, + "unit": "ms", + "iterations": len(cold_times), + }, + {"metric": "cold_start_p99_ms", "value": cold_stats.p99, "unit": "ms"}, + ] + ) # 6. Request latency benchmark - req_times = asyncio.run(benchmark_request_processing(min(args.handlers, 1000), 10000, verbose)) + req_times = asyncio.run( + benchmark_request_processing(min(args.handlers, 1000), 10000, verbose) + ) req_stats = calculate_stats(req_times) suite["raw_data"]["request_latency_us"] = req_times - suite["results"].extend([ - {"metric": "request_latency_mean_us", "value": req_stats.mean, "unit": "us"}, - {"metric": "request_latency_p50_us", "value": req_stats.p50, "unit": "us"}, - {"metric": "request_latency_p95_us", "value": req_stats.p95, "unit": "us"}, - {"metric": "request_latency_p99_us", "value": req_stats.p99, "unit": "us"}, - ]) + suite["results"].extend( + [ + { + "metric": "request_latency_mean_us", + "value": req_stats.mean, + "unit": "us", + }, + {"metric": "request_latency_p50_us", "value": req_stats.p50, "unit": "us"}, + {"metric": "request_latency_p95_us", "value": req_stats.p95, "unit": "us"}, + {"metric": "request_latency_p99_us", "value": req_stats.p99, "unit": "us"}, + ] + ) if req_stats.mean > 0: - suite["results"].append({ - "metric": "theoretical_single_thread_rps", - "value": 1_000_000 / req_stats.mean, - "unit": "req/s", - }) + suite["results"].append( + { + "metric": "theoretical_single_thread_rps", + "value": 1_000_000 / req_stats.mean, + "unit": "req/s", + } + ) if args.json: print(json.dumps(suite, indent=2)) @@ -406,8 +496,12 @@ def main(): print("\n=== Summary ===") print(f"Agent Init: {agent_init_stats.mean:.2f} ms (one-time overhead)") print(f"Agent Memory: {agent_mem_stats.mean:.2f} MB (one-time overhead)") - print(f"Handler Registration ({reg_handlers}): {reg_stats.mean:.2f} ms ({reg_stats.mean / reg_handlers:.3f} ms/handler)") - print(f"Handler Memory ({mem_handlers}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_handlers:.0f} bytes/handler)") + print( + f"Handler Registration ({reg_handlers}): {reg_stats.mean:.2f} ms ({reg_stats.mean / reg_handlers:.3f} ms/handler)" + ) + print( + f"Handler Memory ({mem_handlers}): {mem_stats.mean:.2f} MB ({(mem_stats.mean * 1024 * 1024) / mem_handlers:.0f} bytes/handler)" + ) print(f"Cold Start (Agent + 1 handler): {cold_stats.mean:.2f} ms") print(f"Request Latency p99: {req_stats.p99:.2f} µs") diff --git a/examples/e2e_resilience_tests/agent_flaky.py b/examples/e2e_resilience_tests/agent_flaky.py index 5bfdd14c8..5b4dd3aed 100644 --- a/examples/e2e_resilience_tests/agent_flaky.py +++ b/examples/e2e_resilience_tests/agent_flaky.py @@ -15,8 +15,6 @@ ai_config=AIConfig( model=os.getenv("SMALL_MODEL", "openai/gpt-4o-mini"), temperature=0.0 ), - - ) diff --git a/examples/e2e_resilience_tests/mock_llm_server.py b/examples/e2e_resilience_tests/mock_llm_server.py index c18a522ea..34274410c 100644 --- a/examples/e2e_resilience_tests/mock_llm_server.py +++ b/examples/e2e_resilience_tests/mock_llm_server.py @@ -73,7 +73,9 @@ def _handle_health(self): if STATE["frozen"]: pass # Will hang below elif STATE["error"]: - self._respond(500, {"status": "error", "message": "simulated LLM failure"}) + self._respond( + 500, {"status": "error", "message": "simulated LLM failure"} + ) return else: self._respond(200, {"status": "ok"}) @@ -87,12 +89,15 @@ def _handle_models(self): if STATE["frozen"]: pass # hang else: - self._respond(200, { - "data": [ - {"id": "gpt-4o-mini", "object": "model"}, - {"id": "gpt-4o", "object": "model"}, - ] - }) + self._respond( + 200, + { + "data": [ + {"id": "gpt-4o-mini", "object": "model"}, + {"id": "gpt-4o", "object": "model"}, + ] + }, + ) return self._hang_until_unfrozen() @@ -104,19 +109,28 @@ def _handle_chat(self): self._respond(500, {"error": {"message": "LLM backend error"}}) return else: - self._respond(200, { - "id": "mock-resp-001", - "object": "chat.completion", - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": "This is a mock LLM response." + self._respond( + 200, + { + "id": "mock-resp-001", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a mock LLM response.", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, }, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} - }) + }, + ) return self._hang_until_unfrozen() @@ -124,22 +138,35 @@ def _handle_freeze(self): with STATE_LOCK: STATE["frozen"] = True STATE["freeze_count"] += 1 - self._respond(200, {"action": "frozen", "message": "LLM server is now hanging on all requests"}) + self._respond( + 200, + { + "action": "frozen", + "message": "LLM server is now hanging on all requests", + }, + ) def _handle_unfreeze(self): with STATE_LOCK: STATE["frozen"] = False - self._respond(200, {"action": "unfrozen", "message": "LLM server resumed normal operation"}) + self._respond( + 200, + {"action": "unfrozen", "message": "LLM server resumed normal operation"}, + ) def _handle_set_error(self): with STATE_LOCK: STATE["error"] = True - self._respond(200, {"action": "error_mode", "message": "LLM /health now returns 500"}) + self._respond( + 200, {"action": "error_mode", "message": "LLM /health now returns 500"} + ) def _handle_recover(self): with STATE_LOCK: STATE["error"] = False - self._respond(200, {"action": "recovered", "message": "LLM /health now returns 200"}) + self._respond( + 200, {"action": "recovered", "message": "LLM /health now returns 200"} + ) def _handle_state(self): with STATE_LOCK: @@ -165,8 +192,11 @@ def _respond(self, status, body): class ThreadedHTTPServer(HTTPServer): """Handle requests in a separate thread so freeze doesn't block control endpoints.""" + def process_request(self, request, client_address): - t = threading.Thread(target=self._handle_request_thread, args=(request, client_address)) + t = threading.Thread( + target=self._handle_request_thread, args=(request, client_address) + ) t.daemon = True t.start() diff --git a/examples/python_agent_nodes/docker_hello_world/main.py b/examples/python_agent_nodes/docker_hello_world/main.py index 80ddd877c..3a72cf6c8 100644 --- a/examples/python_agent_nodes/docker_hello_world/main.py +++ b/examples/python_agent_nodes/docker_hello_world/main.py @@ -49,4 +49,3 @@ def _log_heartbeat() -> None: # For containerized runs, set AGENT_CALLBACK_URL so the control plane can call back: # AGENT_CALLBACK_URL=http://: app.run(host="0.0.0.0", port=port, auto_port=False) - diff --git a/examples/python_agent_nodes/documentation_chatbot/routers/ingestion.py b/examples/python_agent_nodes/documentation_chatbot/routers/ingestion.py index 6b1e2e426..c88b56aeb 100644 --- a/examples/python_agent_nodes/documentation_chatbot/routers/ingestion.py +++ b/examples/python_agent_nodes/documentation_chatbot/routers/ingestion.py @@ -28,7 +28,9 @@ async def _clear_namespace_via_api(namespace: str) -> dict: base_url = os.getenv("CONTROL_PLANE_URL") or os.getenv("AGENTFIELD_SERVER") or "" base_url = base_url.rstrip("/") if not base_url: - raise ValueError("CONTROL_PLANE_URL (or AGENTFIELD_SERVER) is required to clear namespace") + raise ValueError( + "CONTROL_PLANE_URL (or AGENTFIELD_SERVER) is required to clear namespace" + ) api_key = os.getenv("CONTROL_PLANE_API_KEY") or os.getenv("AGENTFIELD_API_KEY") headers = {"Content-Type": "application/json"} @@ -37,7 +39,9 @@ async def _clear_namespace_via_api(namespace: str) -> dict: url = f"{base_url}/api/v1/memory/vector/namespace" async with httpx.AsyncClient(timeout=30) as client: - resp = await client.delete(url, json={"namespace": namespace, "scope": "global"}, headers=headers) + resp = await client.delete( + url, json={"namespace": namespace, "scope": "global"}, headers=headers + ) resp.raise_for_status() return resp.json() diff --git a/examples/python_agent_nodes/documentation_chatbot/routers/qa.py b/examples/python_agent_nodes/documentation_chatbot/routers/qa.py index f3325739a..a71a48db0 100644 --- a/examples/python_agent_nodes/documentation_chatbot/routers/qa.py +++ b/examples/python_agent_nodes/documentation_chatbot/routers/qa.py @@ -59,7 +59,7 @@ async def synthesize_answer( ## CITATION FORMAT (CRITICAL) -You have access to these source keys: {', '.join(citation_keys)} +You have access to these source keys: {", ".join(citation_keys)} **How to cite in your answer text:** - Write the citation key wrapped in square brackets: [A], [B], [C], etc. @@ -282,7 +282,7 @@ async def qa_answer_with_documents( ## CITATION FORMAT (CRITICAL) -You have access to these source keys: {', '.join(citation_keys)} +You have access to these source keys: {", ".join(citation_keys)} **How to cite in your answer text:** - Write the citation key wrapped in square brackets: [A], [B], [C], etc. diff --git a/examples/python_agent_nodes/multi_version/main.py b/examples/python_agent_nodes/multi_version/main.py index da426d8fe..a890c89ec 100644 --- a/examples/python_agent_nodes/multi_version/main.py +++ b/examples/python_agent_nodes/multi_version/main.py @@ -138,7 +138,9 @@ def main(): print(f" Control plane: {CP_URL}") print(f" Agent ID: {AGENT_ID}") print( - " Versions: " + ", ".join(f"{v['version']}@:{v['port']}" for v in versions) + "\n" + " Versions: " + + ", ".join(f"{v['version']}@:{v['port']}" for v in versions) + + "\n" ) # Create and start all agents in background threads diff --git a/examples/python_agent_nodes/permission_agent_a/main.py b/examples/python_agent_nodes/permission_agent_a/main.py index cf62d925c..5924f7a75 100644 --- a/examples/python_agent_nodes/permission_agent_a/main.py +++ b/examples/python_agent_nodes/permission_agent_a/main.py @@ -95,5 +95,7 @@ async def call_delete(table: str = "sensitive_records") -> dict: print("Permission Agent A (Caller)") print("Node: permission-agent-a") print("Tags: analytics") - print("Test reasoners: call_query_data (allow), call_query_large (constraint), call_delete (deny)") + print( + "Test reasoners: call_query_data (allow), call_query_large (constraint), call_delete (deny)" + ) app.run(auto_port=True) diff --git a/examples/python_agent_nodes/rag_evaluation/main.py b/examples/python_agent_nodes/rag_evaluation/main.py index 0dbb65d89..5ac1e8dba 100644 --- a/examples/python_agent_nodes/rag_evaluation/main.py +++ b/examples/python_agent_nodes/rag_evaluation/main.py @@ -15,6 +15,7 @@ # Load .env file for local development try: from dotenv import load_dotenv + load_dotenv() except ImportError: pass # python-dotenv not installed, use environment variables directly diff --git a/examples/python_agent_nodes/rag_evaluation/ml_services/embeddings.py b/examples/python_agent_nodes/rag_evaluation/ml_services/embeddings.py index 0063cb619..f80424a41 100644 --- a/examples/python_agent_nodes/rag_evaluation/ml_services/embeddings.py +++ b/examples/python_agent_nodes/rag_evaluation/ml_services/embeddings.py @@ -22,7 +22,8 @@ def _get_model(): if _model is None: try: from sentence_transformers import SentenceTransformer - _model = SentenceTransformer('all-MiniLM-L6-v2') + + _model = SentenceTransformer("all-MiniLM-L6-v2") except ImportError: raise ImportError( "sentence-transformers required. Install with: " @@ -95,11 +96,7 @@ def batch_similarity(self, query: str, candidates: List[str]) -> List[float]: return scores.tolist() def find_most_similar( - self, - query: str, - candidates: List[str], - top_k: int = 3, - threshold: float = 0.5 + self, query: str, candidates: List[str], top_k: int = 3, threshold: float = 0.5 ) -> List[tuple]: """ Find most similar candidates to query. diff --git a/examples/python_agent_nodes/rag_evaluation/ml_services/ner.py b/examples/python_agent_nodes/rag_evaluation/ml_services/ner.py index 292d03f21..d0172aac5 100644 --- a/examples/python_agent_nodes/rag_evaluation/ml_services/ner.py +++ b/examples/python_agent_nodes/rag_evaluation/ml_services/ner.py @@ -18,11 +18,13 @@ def _get_spacy(): if _nlp is None: try: import spacy + try: _nlp = spacy.load("en_core_web_sm") except OSError: # Model not installed, try downloading import subprocess + subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"]) _nlp = spacy.load("en_core_web_sm") except ImportError: @@ -66,12 +68,14 @@ def extract_entities(self, text: str) -> List[Dict]: entities = [] for ent in doc.ents: - entities.append({ - "text": ent.text, - "label": ent.label_, - "start": ent.start_char, - "end": ent.end_char - }) + entities.append( + { + "text": ent.text, + "label": ent.label_, + "start": ent.start_char, + "end": ent.end_char, + } + ) return entities @@ -92,7 +96,7 @@ def extract_factual_claims(self, text: str) -> List[str]: # Check if sentence contains entities or numbers sent_doc = self.nlp(sent.text) has_entities = len(sent_doc.ents) > 0 - has_numbers = bool(re.search(r'\d+', sent.text)) + has_numbers = bool(re.search(r"\d+", sent.text)) if has_entities or has_numbers: factual_sentences.append(sent.text.strip()) @@ -119,11 +123,9 @@ def extract_numbers_with_context(self, text: str) -> List[Dict]: end = min(len(doc), token.i + 6) context = doc[start:end].text - numbers.append({ - "number": token.text, - "context": context, - "position": token.idx - }) + numbers.append( + {"number": token.text, "context": context, "position": token.idx} + ) return numbers @@ -148,7 +150,7 @@ def split_into_claims(self, text: str) -> List[str]: # Check for compound sentences with conjunctions if " and " in sent_text.lower() or " but " in sent_text.lower(): # Try to split on conjunctions while preserving meaning - parts = re.split(r'\s+(?:and|but)\s+', sent_text, flags=re.IGNORECASE) + parts = re.split(r"\s+(?:and|but)\s+", sent_text, flags=re.IGNORECASE) for part in parts: part = part.strip() if len(part) > 10: # Skip very short fragments @@ -159,10 +161,7 @@ def split_into_claims(self, text: str) -> List[str]: return claims def find_entity_in_text( - self, - entity: str, - text: str, - fuzzy: bool = False + self, entity: str, text: str, fuzzy: bool = False ) -> Optional[Dict]: """ Find an entity mention in text. @@ -182,7 +181,7 @@ def find_entity_in_text( "found": True, "match_type": "exact", "position": idx, - "matched_text": text[idx:idx+len(entity)] + "matched_text": text[idx : idx + len(entity)], } if fuzzy: @@ -192,12 +191,15 @@ def find_entity_in_text( for ent in doc.ents: # Check similarity - if ent.text.lower() in entity.lower() or entity.lower() in ent.text.lower(): + if ( + ent.text.lower() in entity.lower() + or entity.lower() in ent.text.lower() + ): return { "found": True, "match_type": "fuzzy", "position": ent.start_char, - "matched_text": ent.text + "matched_text": ent.text, } return None diff --git a/examples/python_agent_nodes/rag_evaluation/ml_services/nli.py b/examples/python_agent_nodes/rag_evaluation/ml_services/nli.py index 7b25c5d60..2ee98e9ee 100644 --- a/examples/python_agent_nodes/rag_evaluation/ml_services/nli.py +++ b/examples/python_agent_nodes/rag_evaluation/ml_services/nli.py @@ -63,11 +63,7 @@ def _ensure_model(self): if self._model is None: self._model, self._tokenizer = _get_model() - def check_entailment( - self, - premise: str, - hypothesis: str - ) -> Dict[str, float]: + def check_entailment(self, premise: str, hypothesis: str) -> Dict[str, float]: """ Check entailment between premise and hypothesis. @@ -83,11 +79,7 @@ def check_entailment( # Tokenize inputs = self._tokenizer( - premise, - hypothesis, - truncation=True, - max_length=512, - return_tensors="pt" + premise, hypothesis, truncation=True, max_length=512, return_tensors="pt" ) # Inference @@ -107,14 +99,12 @@ def check_entailment( "all_scores": { "contradiction": float(scores[0]), "neutral": float(scores[1]), - "entailment": float(scores[2]) - } + "entailment": float(scores[2]), + }, } def batch_check_entailment( - self, - premise: str, - hypotheses: List[str] + self, premise: str, hypotheses: List[str] ) -> List[Dict[str, float]]: """ Check entailment for multiple hypotheses against same premise. @@ -126,17 +116,14 @@ def batch_check_entailment( Returns: List of entailment results """ - return [ - self.check_entailment(premise, h) - for h in hypotheses - ] + return [self.check_entailment(premise, h) for h in hypotheses] def verify_claim( self, context: str, claim: str, entailment_threshold: float = 0.7, - contradiction_threshold: float = 0.7 + contradiction_threshold: float = 0.7, ) -> Dict: """ Verify a claim against context with configurable thresholds. @@ -155,7 +142,10 @@ def verify_claim( if result["label"] == "entailment" and result["score"] >= entailment_threshold: status = "verified" confidence = result["score"] - elif result["label"] == "contradiction" and result["score"] >= contradiction_threshold: + elif ( + result["label"] == "contradiction" + and result["score"] >= contradiction_threshold + ): status = "contradicted" confidence = result["score"] elif result["all_scores"]["entailment"] > result["all_scores"]["contradiction"]: @@ -172,7 +162,7 @@ def verify_claim( "status": status, "confidence": confidence, "needs_llm_review": status.startswith("uncertain"), - "raw_result": result + "raw_result": result, } diff --git a/examples/python_agent_nodes/rag_evaluation/models.py b/examples/python_agent_nodes/rag_evaluation/models.py index 5a99ae2c9..91b2b7853 100644 --- a/examples/python_agent_nodes/rag_evaluation/models.py +++ b/examples/python_agent_nodes/rag_evaluation/models.py @@ -13,21 +13,28 @@ # INPUT MODELS # ============================================ + class EvaluationInput(BaseModel): """Input for RAG evaluation.""" + question: str = Field(description="The user's question") context: str = Field(description="Retrieved context from RAG system") response: str = Field(description="RAG-generated response to evaluate") sources: Optional[List[str]] = Field(None, description="Optional source documents") - model: Optional[str] = Field(None, description="Model to use for evaluation (e.g., 'openrouter/deepseek/deepseek-chat-v3-0324')") + model: Optional[str] = Field( + None, + description="Model to use for evaluation (e.g., 'openrouter/deepseek/deepseek-chat-v3-0324')", + ) # ============================================ # FAITHFULNESS MODELS (Adversarial Debate) # ============================================ + class AtomicClaim(BaseModel): """A single factual claim extracted from response.""" + claim_text: str = Field(description="The factual assertion") claim_type: Literal["factual", "inferential", "opinion"] importance: Literal["critical", "supporting", "minor"] @@ -35,12 +42,14 @@ class AtomicClaim(BaseModel): class ClaimExtraction(BaseModel): """Output from claim extractor.""" + claims: List[AtomicClaim] total_count: int = Field(description="Number of claims extracted") class ProsecutorAttack(BaseModel): """Prosecutor's attack on a claim.""" + claim_index: int = Field(description="Which claim is being attacked") attack_type: Literal["unsupported", "contradicted", "exaggerated", "out_of_context"] evidence: str = Field(description="Why this claim is unfaithful") @@ -49,26 +58,35 @@ class ProsecutorAttack(BaseModel): class ProsecutorAnalysis(BaseModel): """Full prosecutor analysis output.""" + attacks: List[ProsecutorAttack] prosecution_summary: str = Field(description="Overall prosecution case") class DefenderDefense(BaseModel): """Defender's defense of a claim.""" + claim_index: int = Field(description="Which claim is being defended") - defense_type: Literal["direct_support", "implicit_support", "reasonable_inference", "acknowledged_issue"] + defense_type: Literal[ + "direct_support", + "implicit_support", + "reasonable_inference", + "acknowledged_issue", + ] evidence: str = Field(description="Evidence supporting the claim") strength: float = Field(ge=0.0, le=1.0, description="Strength of defense") class DefenderAnalysis(BaseModel): """Full defender analysis output.""" + defenses: List[DefenderDefense] defense_summary: str = Field(description="Overall defense case") class FaithfulnessVerdict(BaseModel): """Final faithfulness judgment from judge.""" + score: float = Field(ge=0.0, le=1.0, description="Faithfulness score") unfaithful_claims: List[str] = Field(description="Claims ruled unfaithful") debate_summary: str = Field(description="Key debate points") @@ -79,15 +97,20 @@ class FaithfulnessVerdict(BaseModel): # RELEVANCE MODELS (Multi-Jury Consensus) # ============================================ + class QuestionIntent(BaseModel): """Question analysis output.""" + primary_intent: str = Field(description="What user really wants to know") sub_questions: List[str] = Field(description="Implicit sub-questions") - expected_type: Literal["factual", "explanation", "list", "comparison", "procedure", "opinion"] + expected_type: Literal[ + "factual", "explanation", "list", "comparison", "procedure", "opinion" + ] class JurorVote(BaseModel): """Single juror's vote.""" + dimension: Literal["literal", "intent", "scope"] score: float = Field(ge=0.0, le=1.0) reasoning: str = Field(description="Why this score") @@ -96,11 +119,14 @@ class JurorVote(BaseModel): class RelevanceVerdict(BaseModel): """Final relevance verdict from foreman.""" + overall_score: float = Field(ge=0.0, le=1.0) literal_score: float = Field(ge=0.0, le=1.0) intent_score: float = Field(ge=0.0, le=1.0) scope_score: float = Field(ge=0.0, le=1.0) - disagreement_level: float = Field(ge=0.0, le=1.0, description="How much jurors disagreed") + disagreement_level: float = Field( + ge=0.0, le=1.0, description="How much jurors disagreed" + ) verdict: str = Field(description="Summary verdict") @@ -108,14 +134,17 @@ class RelevanceVerdict(BaseModel): # HALLUCINATION MODELS (Hybrid ML+LLM) # ============================================ + class StatementExtraction(BaseModel): """Extracted factual statements for verification.""" + statements: List[str] = Field(description="Factual statements to verify") entity_count: int = Field(description="Number of named entities found") class MLVerificationResult(BaseModel): """ML-based verification result for a statement.""" + statement_index: int verification_status: Literal["verified", "uncertain", "unverified"] confidence: float = Field(ge=0.0, le=1.0) @@ -124,6 +153,7 @@ class MLVerificationResult(BaseModel): class LLMVerificationResult(BaseModel): """LLM verification for uncertain statements.""" + statement_index: int is_hallucination: bool explanation: str = Field(description="Why this is/isn't a hallucination") @@ -132,7 +162,10 @@ class LLMVerificationResult(BaseModel): class HallucinationReport(BaseModel): """Final hallucination detection report.""" - score: float = Field(ge=0.0, le=1.0, description="Groundedness score (1=fully grounded)") + + score: float = Field( + ge=0.0, le=1.0, description="Groundedness score (1=fully grounded)" + ) fabrications: List[str] = Field(description="Fabricated statements") contradictions: List[str] = Field(description="Statements contradicting context") ml_handled_percent: float = Field(ge=0.0, le=100.0, description="% handled by ML") @@ -143,8 +176,10 @@ class HallucinationReport(BaseModel): # CONSTITUTIONAL MODELS (Principles-Based) # ============================================ + class PrincipleViolation(BaseModel): """A single principle violation.""" + principle_id: str principle_name: str violation: str = Field(description="What violated the principle") @@ -153,6 +188,7 @@ class PrincipleViolation(BaseModel): class PrincipleCheck(BaseModel): """Result of checking one principle.""" + principle_id: str score: float = Field(ge=0.0, le=1.0) passed: bool @@ -162,8 +198,11 @@ class PrincipleCheck(BaseModel): class ConstitutionalReport(BaseModel): """Full constitutional compliance report.""" + overall_score: float = Field(ge=0.0, le=1.0) - compliance_status: Literal["compliant", "minor_issues", "major_issues", "non_compliant"] + compliance_status: Literal[ + "compliant", "minor_issues", "major_issues", "non_compliant" + ] critical_violations: List[PrincipleViolation] principle_scores: Dict[str, float] improvement_needed: List[str] @@ -173,8 +212,10 @@ class ConstitutionalReport(BaseModel): # UNIFIED OUTPUT MODELS # ============================================ + class RAGEvaluationResult(BaseModel): """Complete RAG evaluation result.""" + # Core metric scores faithfulness: FaithfulnessVerdict relevance: RelevanceVerdict @@ -200,8 +241,10 @@ class RAGEvaluationResult(BaseModel): # (For quick mode evaluation) # ============================================ + class QuickFaithfulness(BaseModel): """Simplified faithfulness for quick mode.""" + score: float = Field(ge=0.0, le=1.0) issues: List[str] reasoning: str @@ -209,6 +252,7 @@ class QuickFaithfulness(BaseModel): class QuickRelevance(BaseModel): """Simplified relevance for quick mode.""" + score: float = Field(ge=0.0, le=1.0) addresses_question: bool reasoning: str @@ -216,6 +260,7 @@ class QuickRelevance(BaseModel): class QuickHallucination(BaseModel): """Simplified hallucination for quick mode.""" + score: float = Field(ge=0.0, le=1.0) fabrications_found: List[str] reasoning: str @@ -223,6 +268,7 @@ class QuickHallucination(BaseModel): class QuickConstitutional(BaseModel): """Simplified constitutional for quick mode.""" + score: float = Field(ge=0.0, le=1.0) violations: List[str] reasoning: str @@ -232,11 +278,22 @@ class QuickConstitutional(BaseModel): # CONFIGURATION MODELS # ============================================ + class EvaluationConfig(BaseModel): """Configuration for evaluation run.""" + mode: Literal["quick", "standard", "thorough"] = Field(default="standard") - enable_ml: bool = Field(default=True, description="Use ML models for hallucination detection") - constitution_file: Optional[str] = Field(None, description="Path to custom constitution") - domain: Optional[Literal["general", "medical", "legal", "financial"]] = Field(default="general") + enable_ml: bool = Field( + default=True, description="Use ML models for hallucination detection" + ) + constitution_file: Optional[str] = Field( + None, description="Path to custom constitution" + ) + domain: Optional[Literal["general", "medical", "legal", "financial"]] = Field( + default="general" + ) faithfulness_debate_mode: Literal["full", "simplified"] = Field(default="full") - model: Optional[str] = Field(None, description="Model to use for all AI calls (e.g., 'openrouter/deepseek/deepseek-chat-v3-0324')") + model: Optional[str] = Field( + None, + description="Model to use for all AI calls (e.g., 'openrouter/deepseek/deepseek-chat-v3-0324')", + ) diff --git a/examples/python_agent_nodes/rag_evaluation/rag_eval_client.py b/examples/python_agent_nodes/rag_evaluation/rag_eval_client.py index c59448d0f..850366359 100644 --- a/examples/python_agent_nodes/rag_evaluation/rag_eval_client.py +++ b/examples/python_agent_nodes/rag_evaluation/rag_eval_client.py @@ -34,6 +34,7 @@ class Domain(str, Enum): @dataclass class FaithfulnessResult: """Result from adversarial debate faithfulness evaluation""" + score: float unfaithful_claims: list[str] debate_summary: str @@ -43,6 +44,7 @@ class FaithfulnessResult: @dataclass class RelevanceResult: """Result from multi-jury relevance evaluation""" + overall_score: float literal_score: float intent_score: float @@ -54,6 +56,7 @@ class RelevanceResult: @dataclass class HallucinationResult: """Result from hybrid ML+LLM hallucination detection""" + score: float fabrications: list[str] contradictions: list[str] @@ -64,6 +67,7 @@ class HallucinationResult: @dataclass class ConstitutionalResult: """Result from principles-based constitutional evaluation""" + overall_score: float compliance_status: str critical_violations: list[str] @@ -74,6 +78,7 @@ class ConstitutionalResult: @dataclass class RAGEvaluationResult: """Complete RAG evaluation result with all metrics""" + overall_score: float quality_tier: str evaluation_mode: str @@ -111,7 +116,7 @@ def __init__( self, agentfield_server: str = "http://localhost:8080", timeout: float = 60.0, - agent_id: str = "rag-evaluation" + agent_id: str = "rag-evaluation", ): """ Initialize the RAG Evaluator client. @@ -141,7 +146,7 @@ def evaluate( context: str, response: str, mode: EvaluationMode = EvaluationMode.STANDARD, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> RAGEvaluationResult: """ Perform full RAG evaluation with all 4 metrics. @@ -162,13 +167,16 @@ def evaluate( Returns: RAGEvaluationResult with all metrics and recommendations """ - result = self._execute("evaluate_rag_response", { - "question": question, - "context": context, - "response": response, - "mode": mode.value, - "domain": domain.value - }) + result = self._execute( + "evaluate_rag_response", + { + "question": question, + "context": context, + "response": response, + "mode": mode.value, + "domain": domain.value, + }, + ) return self._parse_full_result(result) @@ -177,7 +185,7 @@ def evaluate_quick( question: str, context: str, response: str, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> RAGEvaluationResult: """Quick evaluation (~4 AI calls) for real-time validation""" return self.evaluate(question, context, response, EvaluationMode.QUICK, domain) @@ -187,28 +195,29 @@ def evaluate_standard( question: str, context: str, response: str, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> RAGEvaluationResult: """Standard evaluation (~14 AI calls) for production use""" - return self.evaluate(question, context, response, EvaluationMode.STANDARD, domain) + return self.evaluate( + question, context, response, EvaluationMode.STANDARD, domain + ) def evaluate_thorough( self, question: str, context: str, response: str, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> RAGEvaluationResult: """Thorough evaluation (~20+ AI calls) for audits/compliance""" - return self.evaluate(question, context, response, EvaluationMode.THOROUGH, domain) + return self.evaluate( + question, context, response, EvaluationMode.THOROUGH, domain + ) # ==================== Individual Metrics ==================== def faithfulness( - self, - response: str, - context: str, - mode: Literal["quick", "full"] = "full" + self, response: str, context: str, mode: Literal["quick", "full"] = "full" ) -> FaithfulnessResult: """ Evaluate faithfulness using adversarial debate. @@ -221,18 +230,14 @@ def faithfulness( context: Source context/documents mode: "quick" (single reasoner) or "full" (adversarial debate) """ - result = self._execute("evaluate_faithfulness_only", { - "response": response, - "context": context, - "mode": mode - }) + result = self._execute( + "evaluate_faithfulness_only", + {"response": response, "context": context, "mode": mode}, + ) return self._parse_faithfulness(result["result"]) def relevance( - self, - question: str, - response: str, - mode: Literal["quick", "full"] = "full" + self, question: str, response: str, mode: Literal["quick", "full"] = "full" ) -> RelevanceResult: """ Evaluate relevance using multi-jury consensus. @@ -247,18 +252,14 @@ def relevance( response: RAG-generated response mode: "quick" (single reasoner) or "full" (multi-jury) """ - result = self._execute("evaluate_relevance_only", { - "question": question, - "response": response, - "mode": mode - }) + result = self._execute( + "evaluate_relevance_only", + {"question": question, "response": response, "mode": mode}, + ) return self._parse_relevance(result["result"]) def hallucination( - self, - response: str, - context: str, - mode: Literal["quick", "full"] = "full" + self, response: str, context: str, mode: Literal["quick", "full"] = "full" ) -> HallucinationResult: """ Detect hallucinations using hybrid ML+LLM approach. @@ -271,11 +272,10 @@ def hallucination( context: Source context/documents mode: "quick" (single reasoner) or "full" (hybrid ML+LLM) """ - result = self._execute("evaluate_hallucination_only", { - "response": response, - "context": context, - "mode": mode - }) + result = self._execute( + "evaluate_hallucination_only", + {"response": response, "context": context, "mode": mode}, + ) return self._parse_hallucination(result["result"]) def constitutional( @@ -284,7 +284,7 @@ def constitutional( response: str, context: str, domain: Domain = Domain.GENERAL, - mode: Literal["quick", "full"] = "full" + mode: Literal["quick", "full"] = "full", ) -> ConstitutionalResult: """ Evaluate constitutional compliance using principles. @@ -303,13 +303,16 @@ def constitutional( domain: Domain preset (general/medical/legal/financial) mode: "quick" (single reasoner) or "full" (parallel principle checks) """ - result = self._execute("evaluate_constitutional_only", { - "question": question, - "response": response, - "context": context, - "domain": domain.value, - "mode": mode - }) + result = self._execute( + "evaluate_constitutional_only", + { + "question": question, + "response": response, + "context": context, + "domain": domain.value, + "mode": mode, + }, + ) return self._parse_constitutional(result["result"]) # ==================== Sampling & Batch Evaluation ==================== @@ -320,7 +323,7 @@ def sample_and_evaluate( sample_size: int = 100, sample_rate: float = None, mode: EvaluationMode = EvaluationMode.QUICK, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> dict: """ Sample RAG logs and evaluate for quality metrics. @@ -353,19 +356,11 @@ def sample_and_evaluate( context=log["context"], response=log["response"], mode=mode, - domain=domain + domain=domain, ) - results.append({ - "input": log, - "result": result, - "success": True - }) + results.append({"input": log, "result": result, "success": True}) except Exception as e: - results.append({ - "input": log, - "error": str(e), - "success": False - }) + results.append({"input": log, "error": str(e), "success": False}) # Compute aggregate statistics successful = [r for r in results if r["success"]] @@ -375,14 +370,31 @@ def sample_and_evaluate( "sample_size": len(samples), "successful_evaluations": len(successful), "failed_evaluations": len(results) - len(successful), - "avg_overall_score": sum(r["result"].overall_score for r in successful) / len(successful), - "avg_faithfulness": sum(r["result"].faithfulness.score for r in successful) / len(successful), - "avg_relevance": sum(r["result"].relevance.overall_score for r in successful) / len(successful), - "avg_hallucination": sum(r["result"].hallucination.score for r in successful) / len(successful), - "avg_constitutional": sum(r["result"].constitutional.overall_score for r in successful) / len(successful), + "avg_overall_score": sum(r["result"].overall_score for r in successful) + / len(successful), + "avg_faithfulness": sum( + r["result"].faithfulness.score for r in successful + ) + / len(successful), + "avg_relevance": sum( + r["result"].relevance.overall_score for r in successful + ) + / len(successful), + "avg_hallucination": sum( + r["result"].hallucination.score for r in successful + ) + / len(successful), + "avg_constitutional": sum( + r["result"].constitutional.overall_score for r in successful + ) + / len(successful), "quality_tier_distribution": self._count_tiers(successful), - "critical_issues_count": sum(len(r["result"].critical_issues) for r in successful), - "requires_human_review_count": sum(1 for r in successful if r["result"].requires_human_review), + "critical_issues_count": sum( + len(r["result"].critical_issues) for r in successful + ), + "requires_human_review_count": sum( + 1 for r in successful if r["result"].requires_human_review + ), } else: stats = { @@ -391,10 +403,7 @@ def sample_and_evaluate( "failed_evaluations": len(results), } - return { - "statistics": stats, - "results": results - } + return {"statistics": stats, "results": results} def _count_tiers(self, results: list) -> dict: """Count quality tier distribution""" @@ -422,7 +431,7 @@ def _parse_full_result(self, api_result: dict) -> RAGEvaluationResult: hallucination=self._parse_hallucination(r.get("hallucination", {})), constitutional=self._parse_constitutional(r.get("constitutional", {})), execution_id=api_result.get("execution_id", ""), - duration_ms=api_result.get("duration_ms", 0) + duration_ms=api_result.get("duration_ms", 0), ) def _parse_faithfulness(self, data: dict) -> FaithfulnessResult: @@ -430,7 +439,7 @@ def _parse_faithfulness(self, data: dict) -> FaithfulnessResult: score=data.get("score", 0), unfaithful_claims=data.get("unfaithful_claims", []), debate_summary=data.get("debate_summary", ""), - reasoning=data.get("reasoning", "") + reasoning=data.get("reasoning", ""), ) def _parse_relevance(self, data: dict) -> RelevanceResult: @@ -440,7 +449,7 @@ def _parse_relevance(self, data: dict) -> RelevanceResult: intent_score=data.get("intent_score", 0), scope_score=data.get("scope_score", 0), disagreement_level=data.get("disagreement_level", 0), - verdict=data.get("verdict", "") + verdict=data.get("verdict", ""), ) def _parse_hallucination(self, data: dict) -> HallucinationResult: @@ -449,7 +458,7 @@ def _parse_hallucination(self, data: dict) -> HallucinationResult: fabrications=data.get("fabrications", []), contradictions=data.get("contradictions", []), ml_handled_percent=data.get("ml_handled_percent", 0), - total_statements=data.get("total_statements", 0) + total_statements=data.get("total_statements", 0), ) def _parse_constitutional(self, data: dict) -> ConstitutionalResult: @@ -458,7 +467,7 @@ def _parse_constitutional(self, data: dict) -> ConstitutionalResult: compliance_status=data.get("compliance_status", "unknown"), critical_violations=data.get("critical_violations", []), principle_scores=data.get("principle_scores", {}), - improvement_needed=data.get("improvement_needed", []) + improvement_needed=data.get("improvement_needed", []), ) def close(self): @@ -474,6 +483,7 @@ def __exit__(self, *args): # ==================== Async Client ==================== + class AsyncRAGEvaluator: """ Async Python SDK client for RAG Evaluation. @@ -490,7 +500,7 @@ def __init__( self, agentfield_server: str = "http://localhost:8080", timeout: float = 60.0, - agent_id: str = "rag-evaluation" + agent_id: str = "rag-evaluation", ): self.base_url = agentfield_server.rstrip("/") self.timeout = timeout @@ -510,58 +520,49 @@ async def evaluate( context: str, response: str, mode: EvaluationMode = EvaluationMode.STANDARD, - domain: Domain = Domain.GENERAL + domain: Domain = Domain.GENERAL, ) -> RAGEvaluationResult: """Async full RAG evaluation with all 4 metrics""" - result = await self._execute("evaluate_rag_response", { - "question": question, - "context": context, - "response": response, - "mode": mode.value, - "domain": domain.value - }) + result = await self._execute( + "evaluate_rag_response", + { + "question": question, + "context": context, + "response": response, + "mode": mode.value, + "domain": domain.value, + }, + ) return RAGEvaluator._parse_full_result(None, result) async def faithfulness( - self, - response: str, - context: str, - mode: Literal["quick", "full"] = "full" + self, response: str, context: str, mode: Literal["quick", "full"] = "full" ) -> FaithfulnessResult: """Async faithfulness evaluation""" - result = await self._execute("evaluate_faithfulness_only", { - "response": response, - "context": context, - "mode": mode - }) + result = await self._execute( + "evaluate_faithfulness_only", + {"response": response, "context": context, "mode": mode}, + ) return RAGEvaluator._parse_faithfulness(None, result["result"]) async def relevance( - self, - question: str, - response: str, - mode: Literal["quick", "full"] = "full" + self, question: str, response: str, mode: Literal["quick", "full"] = "full" ) -> RelevanceResult: """Async relevance evaluation""" - result = await self._execute("evaluate_relevance_only", { - "question": question, - "response": response, - "mode": mode - }) + result = await self._execute( + "evaluate_relevance_only", + {"question": question, "response": response, "mode": mode}, + ) return RAGEvaluator._parse_relevance(None, result["result"]) async def hallucination( - self, - response: str, - context: str, - mode: Literal["quick", "full"] = "full" + self, response: str, context: str, mode: Literal["quick", "full"] = "full" ) -> HallucinationResult: """Async hallucination detection""" - result = await self._execute("evaluate_hallucination_only", { - "response": response, - "context": context, - "mode": mode - }) + result = await self._execute( + "evaluate_hallucination_only", + {"response": response, "context": context, "mode": mode}, + ) return RAGEvaluator._parse_hallucination(None, result["result"]) async def constitutional( @@ -570,16 +571,19 @@ async def constitutional( response: str, context: str, domain: Domain = Domain.GENERAL, - mode: Literal["quick", "full"] = "full" + mode: Literal["quick", "full"] = "full", ) -> ConstitutionalResult: """Async constitutional evaluation""" - result = await self._execute("evaluate_constitutional_only", { - "question": question, - "response": response, - "context": context, - "domain": domain.value, - "mode": mode - }) + result = await self._execute( + "evaluate_constitutional_only", + { + "question": question, + "response": response, + "context": context, + "domain": domain.value, + "mode": mode, + }, + ) return RAGEvaluator._parse_constitutional(None, result["result"]) async def close(self): @@ -595,13 +599,14 @@ async def __aexit__(self, *args): # ==================== Convenience Functions ==================== + def evaluate_rag( question: str, context: str, response: str, server: str = "http://localhost:8080", mode: str = "standard", - domain: str = "general" + domain: str = "general", ) -> RAGEvaluationResult: """ One-liner RAG evaluation. @@ -613,9 +618,7 @@ def evaluate_rag( """ with RAGEvaluator(server) as evaluator: return evaluator.evaluate( - question, context, response, - EvaluationMode(mode), - Domain(domain) + question, context, response, EvaluationMode(mode), Domain(domain) ) @@ -626,7 +629,7 @@ def evaluate_rag( result = evaluator.evaluate_quick( question="What is the capital of France?", context="France is a country in Europe. Paris is the capital city of France.", - response="The capital of France is Paris." + response="The capital of France is Paris.", ) print(f"Overall Score: {result.overall_score}") diff --git a/examples/python_agent_nodes/rag_evaluation/reasoners/constitutional.py b/examples/python_agent_nodes/rag_evaluation/reasoners/constitutional.py index 5a8c2cf51..15db9d152 100644 --- a/examples/python_agent_nodes/rag_evaluation/reasoners/constitutional.py +++ b/examples/python_agent_nodes/rag_evaluation/reasoners/constitutional.py @@ -28,36 +28,36 @@ "name": "No Fabrication", "description": "Every factual claim must trace to source material", "weight": 1.0, - "severity_if_violated": "critical" + "severity_if_violated": "critical", }, { "id": "accurate_attribution", "name": "Accurate Attribution", "description": "Information must be attributed to correct sources", "weight": 0.8, - "severity_if_violated": "major" + "severity_if_violated": "major", }, { "id": "completeness", "name": "Completeness", "description": "Answer addresses all aspects of the question", "weight": 0.6, - "severity_if_violated": "minor" + "severity_if_violated": "minor", }, { "id": "safety", "name": "No Harmful Advice", "description": "Must not recommend dangerous or harmful actions", "weight": 1.0, - "severity_if_violated": "critical" + "severity_if_violated": "critical", }, { "id": "uncertainty_expression", "name": "Uncertainty Expression", "description": "Express uncertainty when information is incomplete or ambiguous", "weight": 0.5, - "severity_if_violated": "minor" - } + "severity_if_violated": "minor", + }, ] @@ -70,8 +70,7 @@ def register_constitutional_reasoners(router): @router.skill() def load_constitution( - config_path: Optional[str] = None, - domain: str = "general" + config_path: Optional[str] = None, domain: str = "general" ) -> dict: """ Load evaluation principles from YAML configuration. @@ -84,7 +83,7 @@ def load_constitution( if config_path and os.path.exists(config_path): try: - with open(config_path, 'r') as f: + with open(config_path, "r") as f: config = yaml.safe_load(f) if config.get("principles"): principles = config["principles"] @@ -101,7 +100,7 @@ def load_constitution( return { "principles": principles, "domain": domain, - "principle_count": len(principles) + "principle_count": len(principles), } # ============================================ @@ -115,7 +114,7 @@ async def check_principle( question: str, response: str, context: str, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Generic principle checker reasoner. @@ -129,8 +128,10 @@ async def check_principle( principle_desc = principle.get("description", "") severity = principle.get("severity_if_violated", "major") - router.note(f"Checking principle: {principle_name}...", - tags=["constitutional", principle_id]) + router.note( + f"Checking principle: {principle_name}...", + tags=["constitutional", principle_id], + ) result = await router.ai( f"""Evaluate if this response adheres to the following principle. @@ -159,7 +160,7 @@ async def check_principle( - violations: List of specific violations (can be empty) - reasoning: Explanation of evaluation""", schema=PrincipleCheck, - model=model + model=model, ) # Ensure principle_id is set correctly @@ -170,17 +171,16 @@ async def check_principle( # Include reasoning from principle check reason_snippet = result.reasoning[:50] if result.reasoning else "Evaluated" status = "passed" if result.passed else "flagged" - router.note(f"{principle_name} ({result.score:.0%}): {reason_snippet}...", - tags=["constitutional", principle_id, status]) + router.note( + f"{principle_name} ({result.score:.0%}): {reason_snippet}...", + tags=["constitutional", principle_id, status], + ) return result_dict @router.reasoner() async def check_no_fabrication( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Dedicated checker for No Fabrication principle. @@ -191,16 +191,15 @@ async def check_no_fabrication( "id": "no_fabrication", "name": "No Fabrication", "description": "Every factual claim must trace to source material", - "severity_if_violated": "critical" + "severity_if_violated": "critical", } - return await check_principle(principle, question, response, context, model=model) + return await check_principle( + principle, question, response, context, model=model + ) @router.reasoner() async def check_accurate_attribution( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Dedicated checker for Accurate Attribution principle. @@ -209,16 +208,15 @@ async def check_accurate_attribution( "id": "accurate_attribution", "name": "Accurate Attribution", "description": "Information must be attributed to correct sources", - "severity_if_violated": "major" + "severity_if_violated": "major", } - return await check_principle(principle, question, response, context, model=model) + return await check_principle( + principle, question, response, context, model=model + ) @router.reasoner() async def check_completeness( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Dedicated checker for Completeness principle. @@ -227,16 +225,15 @@ async def check_completeness( "id": "completeness", "name": "Completeness", "description": "Answer addresses all aspects of the question", - "severity_if_violated": "minor" + "severity_if_violated": "minor", } - return await check_principle(principle, question, response, context, model=model) + return await check_principle( + principle, question, response, context, model=model + ) @router.reasoner() async def check_safety( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Dedicated checker for Safety principle. @@ -247,16 +244,15 @@ async def check_safety( "id": "safety", "name": "No Harmful Advice", "description": "Must not recommend dangerous or harmful actions", - "severity_if_violated": "critical" + "severity_if_violated": "critical", } - return await check_principle(principle, question, response, context, model=model) + return await check_principle( + principle, question, response, context, model=model + ) @router.reasoner() async def check_uncertainty_expression( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Dedicated checker for Uncertainty Expression principle. @@ -265,9 +261,11 @@ async def check_uncertainty_expression( "id": "uncertainty_expression", "name": "Uncertainty Expression", "description": "Express uncertainty when information is incomplete", - "severity_if_violated": "minor" + "severity_if_violated": "minor", } - return await check_principle(principle, question, response, context, model=model) + return await check_principle( + principle, question, response, context, model=model + ) # ============================================ # CONSTITUTIONAL AGGREGATION @@ -275,8 +273,7 @@ async def check_uncertainty_expression( @router.reasoner() async def aggregate_constitutional( - principle_results: List[Dict], - domain: str = "general" + principle_results: List[Dict], domain: str = "general" ) -> dict: """ Aggregate principle check results into final report. @@ -284,8 +281,10 @@ async def aggregate_constitutional( Weighs principle scores and determines overall compliance. """ - router.note("Aggregating constitutional evaluation...", - tags=["constitutional", "aggregation"]) + router.note( + "Aggregating constitutional evaluation...", + tags=["constitutional", "aggregation"], + ) # Load weights constitution = load_constitution(domain=domain) @@ -314,7 +313,9 @@ async def aggregate_constitutional( # Note improvements needed if score < 0.7: - improvement_needed.append(f"{result.get('principle_name', pid)}: {result.get('reasoning', 'Needs improvement')}") + improvement_needed.append( + f"{result.get('principle_name', pid)}: {result.get('reasoning', 'Needs improvement')}" + ) overall_score = weighted_sum / total_weight if total_weight > 0 else 0.5 @@ -333,11 +334,13 @@ async def aggregate_constitutional( compliance_status=status, critical_violations=critical_violations, principle_scores=principle_scores, - improvement_needed=improvement_needed + improvement_needed=improvement_needed, ) - router.note(f"Constitutional: {status} (score: {overall_score:.2f})", - tags=["constitutional", "complete", status]) + router.note( + f"Constitutional: {status} (score: {overall_score:.2f})", + tags=["constitutional", "complete", status], + ) return report.model_dump() @@ -352,7 +355,7 @@ async def evaluate_constitutional_full( context: str, config_path: Optional[str] = None, domain: str = "general", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Full constitutional evaluation orchestrator. @@ -366,44 +369,77 @@ async def evaluate_constitutional_full( orchestrator -> [check_P1, check_P2, check_P3, ...] -> aggregate """ - router.note(f"Starting constitutional compliance check for {domain} domain...", - tags=["constitutional", "orchestration"]) + router.note( + f"Starting constitutional compliance check for {domain} domain...", + tags=["constitutional", "orchestration"], + ) # Load constitution constitution = load_constitution(config_path=config_path, domain=domain) - router.note(f"Evaluating against {constitution['principle_count']} principles in parallel...", - tags=["constitutional", "parallel"]) + router.note( + f"Evaluating against {constitution['principle_count']} principles in parallel...", + tags=["constitutional", "parallel"], + ) # Launch parallel principle checks # Use dedicated checkers for visible workflow nodes tasks = [ - router.app.call("rag-evaluation.check_no_fabrication", - question=question, response=response, context=context, model=model), - router.app.call("rag-evaluation.check_accurate_attribution", - question=question, response=response, context=context, model=model), - router.app.call("rag-evaluation.check_completeness", - question=question, response=response, context=context, model=model), - router.app.call("rag-evaluation.check_safety", - question=question, response=response, context=context, model=model), - router.app.call("rag-evaluation.check_uncertainty_expression", - question=question, response=response, context=context, model=model), + router.app.call( + "rag-evaluation.check_no_fabrication", + question=question, + response=response, + context=context, + model=model, + ), + router.app.call( + "rag-evaluation.check_accurate_attribution", + question=question, + response=response, + context=context, + model=model, + ), + router.app.call( + "rag-evaluation.check_completeness", + question=question, + response=response, + context=context, + model=model, + ), + router.app.call( + "rag-evaluation.check_safety", + question=question, + response=response, + context=context, + model=model, + ), + router.app.call( + "rag-evaluation.check_uncertainty_expression", + question=question, + response=response, + context=context, + model=model, + ), ] # Wait for all parallel checks principle_results = await asyncio.gather(*tasks) - router.note("All principle checks complete", tags=["constitutional", "parallel"]) + router.note( + "All principle checks complete", tags=["constitutional", "parallel"] + ) # Aggregate report = await router.app.call( "rag-evaluation.aggregate_constitutional", principle_results=list(principle_results), - domain=domain + domain=domain, ) - router.note(f"Constitutional complete: {report['compliance_status']}", - tags=["constitutional", "complete"]) + router.note( + f"Constitutional complete: {report['compliance_status']}", + tags=["constitutional", "complete"], + ) return report @@ -413,10 +449,7 @@ async def evaluate_constitutional_full( @router.reasoner() async def evaluate_constitutional_quick( - question: str, - response: str, - context: str, - model: Optional[str] = None + question: str, response: str, context: str, model: Optional[str] = None ) -> dict: """ Quick single-reasoner constitutional check. @@ -448,11 +481,13 @@ async def evaluate_constitutional_quick( - violations: List of principle violations found - reasoning: Brief explanation""", schema=QuickConstitutional, - model=model + model=model, ) - router.note(f"Quick constitutional: {result.score:.2f}", - tags=["constitutional", "quick"]) + router.note( + f"Quick constitutional: {result.score:.2f}", + tags=["constitutional", "quick"], + ) # Determine status if result.score >= 0.8: @@ -467,5 +502,5 @@ async def evaluate_constitutional_quick( compliance_status=status, critical_violations=[], principle_scores={}, - improvement_needed=result.violations + improvement_needed=result.violations, ).model_dump() diff --git a/examples/python_agent_nodes/rag_evaluation/reasoners/faithfulness.py b/examples/python_agent_nodes/rag_evaluation/reasoners/faithfulness.py index e0a2c3fef..93e6ebf2e 100644 --- a/examples/python_agent_nodes/rag_evaluation/reasoners/faithfulness.py +++ b/examples/python_agent_nodes/rag_evaluation/reasoners/faithfulness.py @@ -33,9 +33,7 @@ def register_faithfulness_reasoners(router): @router.reasoner() async def extract_claims( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Claim extractor: Decompose response into atomic claims. @@ -47,7 +45,10 @@ async def extract_claims( Creates the foundation for the adversarial debate. """ - router.note("Extracting atomic claims from response for verification...", tags=["faithfulness", "extraction"]) + router.note( + "Extracting atomic claims from response for verification...", + tags=["faithfulness", "extraction"], + ) result = await router.ai( f"""Analyze this RAG response and extract all factual claims. @@ -62,13 +63,15 @@ async def extract_claims( Extract EVERY verifiable statement as a separate claim.""", schema=ClaimExtraction, - model=model + model=model, ) # Include first claim text as dynamic content first_claim = result.claims[0].claim_text if result.claims else "none found" - router.note(f"Found {result.total_count} claims to verify, starting with: \"{first_claim[:60]}...\"", - tags=["faithfulness", "extraction"]) + router.note( + f'Found {result.total_count} claims to verify, starting with: "{first_claim[:60]}..."', + tags=["faithfulness", "extraction"], + ) return result.model_dump() @@ -78,10 +81,7 @@ async def extract_claims( @router.reasoner() async def prosecute_claims( - claims: List[Dict], - context: str, - response: str, - model: Optional[str] = None + claims: List[Dict], context: str, response: str, model: Optional[str] = None ) -> dict: """ Prosecutor: Adversarial attack on claims. @@ -95,13 +95,17 @@ async def prosecute_claims( The prosecutor's job is to find ALL possible issues. """ - router.note("Prosecutor examining claims for issues...", - tags=["faithfulness", "prosecutor"]) + router.note( + "Prosecutor examining claims for issues...", + tags=["faithfulness", "prosecutor"], + ) - claims_text = "\n".join([ - f"{i+1}. [{c.get('claim_type', 'unknown')}] {c.get('claim_text', c)}" - for i, c in enumerate(claims) - ]) + claims_text = "\n".join( + [ + f"{i + 1}. [{c.get('claim_type', 'unknown')}] {c.get('claim_text', c)}" + for i, c in enumerate(claims) + ] + ) result = await router.ai( f"""You are an aggressive PROSECUTOR. Your job is to find EVERY possible issue @@ -125,14 +129,17 @@ async def prosecute_claims( Be AGGRESSIVE - find every possible issue. Better to over-attack than miss problems. The defender will have a chance to respond.""", schema=ProsecutorAnalysis, - model=model + model=model, ) # Include summary of attack types attack_types = [a.attack_type for a in result.attacks] if result.attacks else [] - summary = result.prosecution_summary[:80] if result.prosecution_summary else "No issues found" - router.note(f"Prosecutor: {summary}...", - tags=["faithfulness", "prosecution"]) + summary = ( + result.prosecution_summary[:80] + if result.prosecution_summary + else "No issues found" + ) + router.note(f"Prosecutor: {summary}...", tags=["faithfulness", "prosecution"]) return result.model_dump() @@ -146,7 +153,7 @@ async def defend_claims( context: str, response: str, attacks: List[Dict], - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Defender: Evidence-based defense of claims. @@ -160,18 +167,24 @@ async def defend_claims( The defender must be fair - acknowledge real problems. """ - router.note("Defender building case for claims...", - tags=["faithfulness", "defender"]) + router.note( + "Defender building case for claims...", tags=["faithfulness", "defender"] + ) - claims_text = "\n".join([ - f"{i+1}. {c.get('claim_text', c)}" - for i, c in enumerate(claims) - ]) + claims_text = "\n".join( + [f"{i + 1}. {c.get('claim_text', c)}" for i, c in enumerate(claims)] + ) - attacks_text = "\n".join([ - f"- Claim {a['claim_index']}: {a['attack_type']} - {a['evidence']}" - for a in attacks - ]) if attacks else "No attacks to defend against." + attacks_text = ( + "\n".join( + [ + f"- Claim {a['claim_index']}: {a['attack_type']} - {a['evidence']}" + for a in attacks + ] + ) + if attacks + else "No attacks to defend against." + ) result = await router.ai( f"""You are a DEFENSE ATTORNEY. Your job is to find evidence supporting these claims @@ -195,13 +208,16 @@ async def defend_claims( Be FAIR - if a claim truly has no support, acknowledge it as 'acknowledged_issue'. Your credibility depends on honesty.""", schema=DefenderAnalysis, - model=model + model=model, ) # Include defense summary - summary = result.defense_summary[:80] if result.defense_summary else "Defense arguments presented" - router.note(f"Defender: {summary}...", - tags=["faithfulness", "defense"]) + summary = ( + result.defense_summary[:80] + if result.defense_summary + else "Defense arguments presented" + ) + router.note(f"Defender: {summary}...", tags=["faithfulness", "defense"]) return result.model_dump() @@ -215,7 +231,7 @@ async def judge_faithfulness( prosecution: Dict, defense: Dict, context: str, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Judge: Impartial verdict on faithfulness. @@ -228,23 +244,33 @@ async def judge_faithfulness( - Calculates overall faithfulness score """ - router.note("Judge deliberating on evidence...", - tags=["faithfulness", "judge"]) + router.note("Judge deliberating on evidence...", tags=["faithfulness", "judge"]) - claims_summary = "\n".join([ - f"{i+1}. {c.get('claim_text', c)}" - for i, c in enumerate(claims) - ]) + claims_summary = "\n".join( + [f"{i + 1}. {c.get('claim_text', c)}" for i, c in enumerate(claims)] + ) - attacks_summary = "\n".join([ - f"- Attack on claim {a['claim_index']}: {a['attack_type']} ({a['severity']}) - {a['evidence']}" - for a in prosecution.get("attacks", []) - ]) if prosecution.get("attacks") else "No attacks." + attacks_summary = ( + "\n".join( + [ + f"- Attack on claim {a['claim_index']}: {a['attack_type']} ({a['severity']}) - {a['evidence']}" + for a in prosecution.get("attacks", []) + ] + ) + if prosecution.get("attacks") + else "No attacks." + ) - defenses_summary = "\n".join([ - f"- Defense of claim {d['claim_index']}: {d['defense_type']} (strength {d['strength']:.2f}) - {d['evidence']}" - for d in defense.get("defenses", []) - ]) if defense.get("defenses") else "No defenses." + defenses_summary = ( + "\n".join( + [ + f"- Defense of claim {d['claim_index']}: {d['defense_type']} (strength {d['strength']:.2f}) - {d['evidence']}" + for d in defense.get("defenses", []) + ] + ) + if defense.get("defenses") + else "No defenses." + ) result = await router.ai( f"""You are an IMPARTIAL JUDGE. Weigh the evidence and render a verdict on faithfulness. @@ -271,13 +297,17 @@ async def judge_faithfulness( Be FAIR and IMPARTIAL. Only rule claims unfaithful if the prosecution proved its case.""", schema=FaithfulnessVerdict, - model=model + model=model, ) # Include key reasoning from judge - reason_snippet = result.reasoning[:80] if result.reasoning else "Verdict reached" - router.note(f"Judge ruling ({result.score:.0%} faithful): {reason_snippet}...", - tags=["faithfulness", "judge", "verdict"]) + reason_snippet = ( + result.reasoning[:80] if result.reasoning else "Verdict reached" + ) + router.note( + f"Judge ruling ({result.score:.0%} faithful): {reason_snippet}...", + tags=["faithfulness", "judge", "verdict"], + ) return result.model_dump() @@ -287,9 +317,7 @@ async def judge_faithfulness( @router.reasoner() async def evaluate_faithfulness_full( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Full adversarial debate orchestrator for faithfulness. @@ -304,15 +332,17 @@ async def evaluate_faithfulness_full( extract_claims -> [prosecutor, defender] -> judge """ - router.note("Starting adversarial debate: prosecutor vs defender...", - tags=["faithfulness", "orchestration"]) + router.note( + "Starting adversarial debate: prosecutor vs defender...", + tags=["faithfulness", "orchestration"], + ) # Step 1: Extract claims claims_result = await router.app.call( "rag-evaluation.extract_claims", response=response, context=context, - model=model + model=model, ) claims = claims_result.get("claims", []) @@ -322,7 +352,7 @@ async def evaluate_faithfulness_full( score=1.0, unfaithful_claims=[], debate_summary="No factual claims found in response", - reasoning="Response contains no verifiable claims" + reasoning="Response contains no verifiable claims", ).model_dump() # Step 2: Prosecutor attacks (can start immediately) @@ -331,7 +361,7 @@ async def evaluate_faithfulness_full( claims=claims, context=context, response=response, - model=model + model=model, ) # Wait for prosecution first (defender needs attacks) @@ -344,7 +374,7 @@ async def evaluate_faithfulness_full( context=context, response=response, attacks=prosecution.get("attacks", []), - model=model + model=model, ) # Step 4: Judge renders verdict @@ -354,11 +384,13 @@ async def evaluate_faithfulness_full( prosecution=prosecution, defense=defense, context=context, - model=model + model=model, ) - router.note(f"Debate complete: {verdict['score']:.2f} faithfulness", - tags=["faithfulness", "complete"]) + router.note( + f"Debate complete: {verdict['score']:.2f} faithfulness", + tags=["faithfulness", "complete"], + ) return verdict @@ -368,9 +400,7 @@ async def evaluate_faithfulness_full( @router.reasoner() async def evaluate_faithfulness_quick( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Quick single-reasoner faithfulness evaluation. @@ -400,16 +430,17 @@ async def evaluate_faithfulness_quick( - issues: List of specific faithfulness issues found - reasoning: Brief explanation""", schema=QuickFaithfulness, - model=model + model=model, ) - router.note(f"Quick faithfulness: {result.score:.2f}", - tags=["faithfulness", "quick"]) + router.note( + f"Quick faithfulness: {result.score:.2f}", tags=["faithfulness", "quick"] + ) # Convert to full verdict format for consistency return FaithfulnessVerdict( score=result.score, unfaithful_claims=result.issues, debate_summary="Quick single-reasoner evaluation (no debate)", - reasoning=result.reasoning + reasoning=result.reasoning, ).model_dump() diff --git a/examples/python_agent_nodes/rag_evaluation/reasoners/hallucination.py b/examples/python_agent_nodes/rag_evaluation/reasoners/hallucination.py index 3585b7d55..dba168ec4 100644 --- a/examples/python_agent_nodes/rag_evaluation/reasoners/hallucination.py +++ b/examples/python_agent_nodes/rag_evaluation/reasoners/hallucination.py @@ -51,23 +51,18 @@ def extract_statements_ml(response: str) -> dict: return { "statements": statements, "entity_count": len(entities), - "entities": entities + "entities": entities, } except ImportError: # Fallback: split into sentences import re - sentences = [s.strip() for s in re.split(r'[.!?]+', response) if s.strip()] - return { - "statements": sentences, - "entity_count": 0, - "entities": [] - } + + sentences = [s.strip() for s in re.split(r"[.!?]+", response) if s.strip()] + return {"statements": sentences, "entity_count": 0, "entities": []} @router.reasoner() async def extract_statements( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Statement extraction reasoner (wraps ML skill). @@ -77,7 +72,9 @@ async def extract_statements( ML service unavailable. """ - router.note("Extracting factual statements...", tags=["hallucination", "extraction"]) + router.note( + "Extracting factual statements...", tags=["hallucination", "extraction"] + ) # Try ML extraction first ml_result = extract_statements_ml(response) @@ -85,13 +82,17 @@ async def extract_statements( if ml_result["statements"]: result = StatementExtraction( statements=ml_result["statements"], - entity_count=ml_result["entity_count"] + entity_count=ml_result["entity_count"], + ) + router.note( + f"ML extracted {len(result.statements)} statements, {result.entity_count} entities", + tags=["hallucination", "ml"], ) - router.note(f"ML extracted {len(result.statements)} statements, {result.entity_count} entities", - tags=["hallucination", "ml"]) else: # Fallback to LLM - router.note("Falling back to LLM extraction...", tags=["hallucination", "fallback"]) + router.note( + "Falling back to LLM extraction...", tags=["hallucination", "fallback"] + ) result = await router.ai( f"""Extract all factual statements from this response that should be verified. @@ -101,13 +102,15 @@ async def extract_statements( List each verifiable factual claim as a separate statement. Focus on claims containing: facts, numbers, dates, names, specific assertions.""", schema=StatementExtraction, - model=model + model=model, ) # Show first statement being checked first_stmt = result.statements[0][:50] if result.statements else "none" - router.note(f"Checking {len(result.statements)} statements, starting with: \"{first_stmt}...\"", - tags=["hallucination", "extraction"]) + router.note( + f'Checking {len(result.statements)} statements, starting with: "{first_stmt}..."', + tags=["hallucination", "extraction"], + ) return result.model_dump() @@ -120,7 +123,7 @@ def ml_verify_statements( statements: List[str], context: str, entailment_threshold: float = 0.7, - similarity_threshold: float = 0.6 + similarity_threshold: float = 0.6, ) -> List[dict]: """ ML-based batch verification skill. @@ -142,7 +145,10 @@ def ml_verify_statements( # Split context into sentences for matching import re - context_sentences = [s.strip() for s in re.split(r'[.!?]+', context) if s.strip()] + + context_sentences = [ + s.strip() for s in re.split(r"[.!?]+", context) if s.strip() + ] for i, statement in enumerate(statements): # Step 1: Find similar context sentences @@ -150,17 +156,19 @@ def ml_verify_statements( statement, context_sentences, top_k=3, - threshold=similarity_threshold + threshold=similarity_threshold, ) if not similar: # No similar sentences found - results.append({ - "statement_index": i, - "verification_status": "unverified", - "confidence": 0.3, - "method": "embedding_similarity" - }) + results.append( + { + "statement_index": i, + "verification_status": "unverified", + "confidence": 0.3, + "method": "embedding_similarity", + } + ) continue # Step 2: NLI entailment check on similar sentences @@ -168,49 +176,54 @@ def ml_verify_statements( nli_result = nli_service.verify_claim( context=best_match[2], # The text claim=statement, - entailment_threshold=entailment_threshold + entailment_threshold=entailment_threshold, ) if nli_result["status"] == "verified": - results.append({ - "statement_index": i, - "verification_status": "verified", - "confidence": nli_result["confidence"], - "method": "nli_entailment" - }) + results.append( + { + "statement_index": i, + "verification_status": "verified", + "confidence": nli_result["confidence"], + "method": "nli_entailment", + } + ) elif nli_result["status"] == "contradicted": - results.append({ - "statement_index": i, - "verification_status": "unverified", - "confidence": nli_result["confidence"], - "method": "nli_entailment" - }) + results.append( + { + "statement_index": i, + "verification_status": "unverified", + "confidence": nli_result["confidence"], + "method": "nli_entailment", + } + ) else: # Uncertain - needs LLM review - results.append({ - "statement_index": i, - "verification_status": "uncertain", - "confidence": nli_result["confidence"], - "method": "nli_entailment" - }) + results.append( + { + "statement_index": i, + "verification_status": "uncertain", + "confidence": nli_result["confidence"], + "method": "nli_entailment", + } + ) except ImportError: # ML services not available - mark all as uncertain for i, _ in enumerate(statements): - results.append({ - "statement_index": i, - "verification_status": "uncertain", - "confidence": 0.5, - "method": "ml_unavailable" - }) + results.append( + { + "statement_index": i, + "verification_status": "uncertain", + "confidence": 0.5, + "method": "ml_unavailable", + } + ) return results @router.reasoner() - async def verify_statements_ml( - statements: List[str], - context: str - ) -> dict: + async def verify_statements_ml(statements: List[str], context: str) -> dict: """ ML verification reasoner (creates workflow node). @@ -218,8 +231,10 @@ async def verify_statements_ml( statements need LLM escalation. """ - router.note(f"ML verifying {len(statements)} statements...", - tags=["hallucination", "ml_verify"]) + router.note( + f"ML verifying {len(statements)} statements...", + tags=["hallucination", "ml_verify"], + ) results = ml_verify_statements(statements, context) @@ -227,15 +242,17 @@ async def verify_statements_ml( unverified = sum(1 for r in results if r["verification_status"] == "unverified") uncertain = sum(1 for r in results if r["verification_status"] == "uncertain") - router.note(f"ML verification: {verified} grounded, {unverified} flagged, {uncertain} need LLM review", - tags=["hallucination", "ml"]) + router.note( + f"ML verification: {verified} grounded, {unverified} flagged, {uncertain} need LLM review", + tags=["hallucination", "ml"], + ) return { "results": results, "verified_count": verified, "unverified_count": unverified, "uncertain_count": uncertain, - "statements": statements + "statements": statements, } # ============================================ @@ -248,7 +265,7 @@ async def verify_statement_llm( statement_index: int, context: str, ml_result: Dict, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ LLM verification for a single uncertain statement. @@ -257,8 +274,10 @@ async def verify_statement_llm( showing the escalation pattern in the graph. """ - router.note(f"LLM verifying statement {statement_index}...", - tags=["hallucination", "llm_verify"]) + router.note( + f"LLM verifying statement {statement_index}...", + tags=["hallucination", "llm_verify"], + ) result = await router.ai( f"""Verify if this statement is a hallucination or grounded in the context. @@ -270,15 +289,15 @@ async def verify_statement_llm( {context} ML PRELIMINARY RESULT: -Status: {ml_result.get('verification_status', 'uncertain')} -Confidence: {ml_result.get('confidence', 0.5):.2f} +Status: {ml_result.get("verification_status", "uncertain")} +Confidence: {ml_result.get("confidence", 0.5):.2f} Determine: - is_hallucination: true if statement contains fabricated/unsupported information - explanation: Why is this a hallucination or why is it grounded - confidence: 0.0-1.0 how confident in this assessment""", schema=LLMVerificationResult, - model=model + model=model, ) # Ensure statement_index is set @@ -288,8 +307,10 @@ async def verify_statement_llm( # Include explanation snippet status = "Fabricated" if result.is_hallucination else "Grounded" explanation = result.explanation[:50] if result.explanation else "" - router.note(f"LLM check #{statement_index}: {status} - {explanation}...", - tags=["hallucination", "llm"]) + router.note( + f"LLM check #{statement_index}: {status} - {explanation}...", + tags=["hallucination", "llm"], + ) return result_dict @@ -298,7 +319,7 @@ async def verify_uncertain_statements( statements: List[str], ml_results: List[Dict], context: str, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ LLM verification orchestrator for uncertain statements. @@ -315,12 +336,16 @@ async def verify_uncertain_statements( ] if not uncertain_indices: - router.note("No uncertain statements to verify", - tags=["hallucination", "llm_verify"]) + router.note( + "No uncertain statements to verify", + tags=["hallucination", "llm_verify"], + ) return {"llm_results": []} - router.note(f"LLM verifying {len(uncertain_indices)} uncertain statements in parallel...", - tags=["hallucination", "llm_parallel"]) + router.note( + f"LLM verifying {len(uncertain_indices)} uncertain statements in parallel...", + tags=["hallucination", "llm_parallel"], + ) # Launch parallel LLM verification tasks = [ @@ -330,7 +355,7 @@ async def verify_uncertain_statements( statement_index=i, context=context, ml_result=ml_results[i], - model=model + model=model, ) for i in uncertain_indices ] @@ -338,8 +363,10 @@ async def verify_uncertain_statements( llm_results = await asyncio.gather(*tasks) hallucinations = sum(1 for r in llm_results if r.get("is_hallucination", False)) - router.note(f"LLM found {hallucinations} hallucinations in {len(llm_results)} uncertain statements", - tags=["hallucination", "llm_complete"]) + router.note( + f"LLM found {hallucinations} hallucinations in {len(llm_results)} uncertain statements", + tags=["hallucination", "llm_complete"], + ) return {"llm_results": list(llm_results)} @@ -352,7 +379,7 @@ async def synthesize_hallucination_report( statements: List[str], ml_results: List[Dict], llm_results: List[Dict], - context: str + context: str, ) -> dict: """ Synthesize final hallucination report. @@ -363,8 +390,9 @@ async def synthesize_hallucination_report( - Cost efficiency metrics (ML vs LLM handling) """ - router.note("Synthesizing hallucination report...", - tags=["hallucination", "synthesis"]) + router.note( + "Synthesizing hallucination report...", tags=["hallucination", "synthesis"] + ) fabrications = [] contradictions = [] @@ -386,7 +414,9 @@ async def synthesize_hallucination_report( # Calculate scores total = len(statements) - ml_handled = sum(1 for r in ml_results if r["verification_status"] != "uncertain") + ml_handled = sum( + 1 for r in ml_results if r["verification_status"] != "uncertain" + ) ml_percent = (ml_handled / total * 100) if total > 0 else 0 hallucination_count = len(fabrications) + len(contradictions) @@ -397,11 +427,13 @@ async def synthesize_hallucination_report( fabrications=fabrications, contradictions=contradictions, ml_handled_percent=ml_percent, - total_statements=total + total_statements=total, ) - router.note(f"Groundedness: {report.score:.2f}, ML handled: {ml_percent:.0f}%", - tags=["hallucination", "complete"]) + router.note( + f"Groundedness: {report.score:.2f}, ML handled: {ml_percent:.0f}%", + tags=["hallucination", "complete"], + ) return report.model_dump() @@ -411,9 +443,7 @@ async def synthesize_hallucination_report( @router.reasoner() async def evaluate_hallucination_full( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Full hybrid ML+LLM orchestrator for hallucination detection. @@ -428,15 +458,17 @@ async def evaluate_hallucination_full( extract -> ml_verify -> [llm_verify_0, llm_verify_1, ...] -> synthesize """ - router.note("Starting hybrid ML + LLM hallucination detection...", - tags=["hallucination", "orchestration"]) + router.note( + "Starting hybrid ML + LLM hallucination detection...", + tags=["hallucination", "orchestration"], + ) # Step 1: Extract statements extraction = await router.app.call( "rag-evaluation.extract_statements", response=response, context=context, - model=model + model=model, ) statements = extraction.get("statements", []) @@ -447,14 +479,14 @@ async def evaluate_hallucination_full( fabrications=[], contradictions=[], ml_handled_percent=100.0, - total_statements=0 + total_statements=0, ).model_dump() # Step 2: ML verification ml_verification = await router.app.call( "rag-evaluation.verify_statements_ml", statements=statements, - context=context + context=context, ) # Step 3: LLM verification for uncertain @@ -463,7 +495,7 @@ async def evaluate_hallucination_full( statements=statements, ml_results=ml_verification.get("results", []), context=context, - model=model + model=model, ) # Step 4: Synthesize @@ -472,11 +504,13 @@ async def evaluate_hallucination_full( statements=statements, ml_results=ml_verification.get("results", []), llm_results=llm_verification.get("llm_results", []), - context=context + context=context, ) - router.note(f"Hybrid detection complete: {report['score']:.2f} groundedness", - tags=["hallucination", "complete"]) + router.note( + f"Hybrid detection complete: {report['score']:.2f} groundedness", + tags=["hallucination", "complete"], + ) return report @@ -486,9 +520,7 @@ async def evaluate_hallucination_full( @router.reasoner() async def evaluate_hallucination_quick( - response: str, - context: str, - model: Optional[str] = None + response: str, context: str, model: Optional[str] = None ) -> dict: """ Quick single-reasoner hallucination check. @@ -517,16 +549,19 @@ async def evaluate_hallucination_quick( - fabrications_found: List of fabricated statements - reasoning: Brief explanation""", schema=QuickHallucination, - model=model + model=model, ) - router.note(f"Quick hallucination: {result.score:.2f}", - tags=["hallucination", "quick"]) + router.note( + f"Quick hallucination: {result.score:.2f}", tags=["hallucination", "quick"] + ) return HallucinationReport( score=result.score, fabrications=result.fabrications_found, contradictions=[], ml_handled_percent=0.0, - total_statements=len(result.fabrications_found) if result.fabrications_found else 1 + total_statements=len(result.fabrications_found) + if result.fabrications_found + else 1, ).model_dump() diff --git a/examples/python_agent_nodes/rag_evaluation/reasoners/orchestrator.py b/examples/python_agent_nodes/rag_evaluation/reasoners/orchestrator.py index ba1a050a2..3770a4e40 100644 --- a/examples/python_agent_nodes/rag_evaluation/reasoners/orchestrator.py +++ b/examples/python_agent_nodes/rag_evaluation/reasoners/orchestrator.py @@ -36,7 +36,7 @@ async def evaluate_rag_response( mode: Literal["quick", "standard", "thorough"] = "standard", domain: str = "general", faithfulness_debate_mode: Literal["full", "simplified"] = "full", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Master RAG evaluation orchestrator. @@ -56,17 +56,22 @@ async def evaluate_rag_response( - Constitutional: principles pattern (check_P1/P2/.../aggregate) """ - router.note(f"Starting {mode} evaluation for domain: {domain}", - tags=["orchestration", "start", mode]) + router.note( + f"Starting {mode} evaluation for domain: {domain}", + tags=["orchestration", "start", mode], + ) # Store evaluation context in workflow memory - await router.memory.set("evaluation_input", { - "question": question, - "context_length": len(context), - "response_length": len(response), - "mode": mode, - "domain": domain - }) + await router.memory.set( + "evaluation_input", + { + "question": question, + "context_length": len(context), + "response_length": len(response), + "mode": mode, + "domain": domain, + }, + ) if mode == "quick": result = await evaluate_quick( @@ -74,7 +79,7 @@ async def evaluate_rag_response( context=context, response=response, domain=domain, - model=model + model=model, ) elif mode == "thorough": result = await evaluate_thorough( @@ -83,7 +88,7 @@ async def evaluate_rag_response( response=response, domain=domain, faithfulness_debate_mode=faithfulness_debate_mode, - model=model + model=model, ) else: # standard result = await evaluate_standard( @@ -92,11 +97,13 @@ async def evaluate_rag_response( response=response, domain=domain, faithfulness_debate_mode=faithfulness_debate_mode, - model=model + model=model, ) - router.note(f"Evaluation complete: {result['quality_tier']} (score: {result['overall_score']:.2f})", - tags=["orchestration", "complete"]) + router.note( + f"Evaluation complete: {result['quality_tier']} (score: {result['overall_score']:.2f})", + tags=["orchestration", "complete"], + ) return result @@ -110,7 +117,7 @@ async def evaluate_quick( context: str, response: str, domain: str = "general", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Quick evaluation mode: Single reasoner per metric. @@ -122,29 +129,31 @@ async def evaluate_quick( Good for: High-volume validation, real-time checks """ - router.note("Quick mode: 4 parallel single-reasoner evaluations", - tags=["orchestration", "quick", "parallel"]) + router.note( + "Quick mode: 4 parallel single-reasoner evaluations", + tags=["orchestration", "quick", "parallel"], + ) # Launch all 4 evaluations in parallel faithfulness_task = router.app.call( "rag-evaluation.evaluate_faithfulness_quick", response=response, context=context, - model=model + model=model, ) relevance_task = router.app.call( "rag-evaluation.evaluate_relevance_quick", question=question, response=response, - model=model + model=model, ) hallucination_task = router.app.call( "rag-evaluation.evaluate_hallucination_quick", response=response, context=context, - model=model + model=model, ) constitutional_task = router.app.call( @@ -152,7 +161,7 @@ async def evaluate_quick( question=question, response=response, context=context, - model=model + model=model, ) # Wait for all @@ -169,7 +178,7 @@ async def evaluate_quick( hallucination=hallucination, constitutional=constitutional, mode="quick", - ai_calls=4 + ai_calls=4, ) # ============================================ @@ -183,7 +192,7 @@ async def evaluate_standard( response: str, domain: str = "general", faithfulness_debate_mode: str = "full", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Standard evaluation mode: Multi-reasoner patterns. @@ -195,8 +204,10 @@ async def evaluate_standard( Good for: Production evaluation, quality assurance """ - router.note("Standard mode: Multi-reasoner patterns", - tags=["orchestration", "standard", "parallel"]) + router.note( + "Standard mode: Multi-reasoner patterns", + tags=["orchestration", "standard", "parallel"], + ) # Choose faithfulness depth if faithfulness_debate_mode == "full": @@ -204,14 +215,14 @@ async def evaluate_standard( "rag-evaluation.evaluate_faithfulness_full", response=response, context=context, - model=model + model=model, ) else: faithfulness_task = router.app.call( "rag-evaluation.evaluate_faithfulness_quick", response=response, context=context, - model=model + model=model, ) # Multi-jury relevance @@ -219,7 +230,7 @@ async def evaluate_standard( "rag-evaluation.evaluate_relevance_full", question=question, response=response, - model=model + model=model, ) # Hybrid hallucination @@ -227,7 +238,7 @@ async def evaluate_standard( "rag-evaluation.evaluate_hallucination_full", response=response, context=context, - model=model + model=model, ) # Constitutional with parallel principle checks @@ -237,7 +248,7 @@ async def evaluate_standard( response=response, context=context, domain=domain, - model=model + model=model, ) # Wait for all @@ -259,7 +270,7 @@ async def evaluate_standard( hallucination=hallucination, constitutional=constitutional, mode="standard", - ai_calls=ai_calls + ai_calls=ai_calls, ) # ============================================ @@ -273,7 +284,7 @@ async def evaluate_thorough( response: str, domain: str = "general", faithfulness_debate_mode: str = "full", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Thorough evaluation mode: Maximum depth on all metrics. @@ -285,29 +296,31 @@ async def evaluate_thorough( Good for: High-stakes evaluation, audits, compliance checks """ - router.note("Thorough mode: Maximum depth evaluation", - tags=["orchestration", "thorough", "parallel"]) + router.note( + "Thorough mode: Maximum depth evaluation", + tags=["orchestration", "thorough", "parallel"], + ) # All metrics at full depth, in parallel faithfulness_task = router.app.call( "rag-evaluation.evaluate_faithfulness_full", response=response, context=context, - model=model + model=model, ) relevance_task = router.app.call( "rag-evaluation.evaluate_relevance_full", question=question, response=response, - model=model + model=model, ) hallucination_task = router.app.call( "rag-evaluation.evaluate_hallucination_full", response=response, context=context, - model=model + model=model, ) constitutional_task = router.app.call( @@ -316,7 +329,7 @@ async def evaluate_thorough( response=response, context=context, domain=domain, - model=model + model=model, ) # Wait for all @@ -332,7 +345,7 @@ async def evaluate_thorough( hallucination=hallucination, constitutional=constitutional, mode="thorough", - ai_calls=20 # Approximate + ai_calls=20, # Approximate ) # ============================================ @@ -341,10 +354,7 @@ async def evaluate_thorough( @router.reasoner() async def evaluate_faithfulness_only( - response: str, - context: str, - mode: str = "full", - model: Optional[str] = None + response: str, context: str, mode: str = "full", model: Optional[str] = None ) -> dict: """ Evaluate faithfulness only. @@ -358,22 +368,19 @@ async def evaluate_faithfulness_only( "rag-evaluation.evaluate_faithfulness_full", response=response, context=context, - model=model + model=model, ) else: return await router.app.call( "rag-evaluation.evaluate_faithfulness_quick", response=response, context=context, - model=model + model=model, ) @router.reasoner() async def evaluate_relevance_only( - question: str, - response: str, - mode: str = "full", - model: Optional[str] = None + question: str, response: str, mode: str = "full", model: Optional[str] = None ) -> dict: """ Evaluate relevance only. @@ -387,43 +394,42 @@ async def evaluate_relevance_only( "rag-evaluation.evaluate_relevance_full", question=question, response=response, - model=model + model=model, ) else: return await router.app.call( "rag-evaluation.evaluate_relevance_quick", question=question, response=response, - model=model + model=model, ) @router.reasoner() async def evaluate_hallucination_only( - response: str, - context: str, - mode: str = "full", - model: Optional[str] = None + response: str, context: str, mode: str = "full", model: Optional[str] = None ) -> dict: """ Evaluate hallucination only. Standalone endpoint for focused hallucination detection. """ - router.note("Hallucination-only evaluation", tags=["hallucination", "standalone"]) + router.note( + "Hallucination-only evaluation", tags=["hallucination", "standalone"] + ) if mode == "full": return await router.app.call( "rag-evaluation.evaluate_hallucination_full", response=response, context=context, - model=model + model=model, ) else: return await router.app.call( "rag-evaluation.evaluate_hallucination_quick", response=response, context=context, - model=model + model=model, ) @router.reasoner() @@ -433,14 +439,16 @@ async def evaluate_constitutional_only( context: str, domain: str = "general", mode: str = "full", - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Evaluate constitutional compliance only. Standalone endpoint for focused constitutional evaluation. """ - router.note("Constitutional-only evaluation", tags=["constitutional", "standalone"]) + router.note( + "Constitutional-only evaluation", tags=["constitutional", "standalone"] + ) if mode == "full": return await router.app.call( @@ -449,7 +457,7 @@ async def evaluate_constitutional_only( response=response, context=context, domain=domain, - model=model + model=model, ) else: return await router.app.call( @@ -457,7 +465,7 @@ async def evaluate_constitutional_only( question=question, response=response, context=context, - model=model + model=model, ) # ============================================ @@ -470,7 +478,7 @@ def aggregate_results( hallucination: Dict, constitutional: Dict, mode: str, - ai_calls: int + ai_calls: int, ) -> dict: """ Aggregate metric results into unified evaluation report. @@ -482,12 +490,7 @@ def aggregate_results( c_score = constitutional.get("overall_score", 0.5) # Weighted average (faithfulness and hallucination weighted higher) - overall = ( - f_score * 0.30 + - r_score * 0.20 + - h_score * 0.30 + - c_score * 0.20 - ) + overall = f_score * 0.30 + r_score * 0.20 + h_score * 0.30 + c_score * 0.20 # Determine quality tier if overall >= 0.9: @@ -504,7 +507,9 @@ def aggregate_results( # Collect critical issues critical_issues = [] if f_score < 0.5: - critical_issues.append("Low faithfulness - response may not be grounded in context") + critical_issues.append( + "Low faithfulness - response may not be grounded in context" + ) if h_score < 0.5: critical_issues.append("Significant hallucinations detected") if constitutional.get("compliance_status") == "non_compliant": @@ -512,9 +517,9 @@ def aggregate_results( # Human review needed? needs_review = ( - overall < 0.5 or - len(critical_issues) > 0 or - relevance.get("disagreement_level", 0) > 0.3 + overall < 0.5 + or len(critical_issues) > 0 + or relevance.get("disagreement_level", 0) > 0.3 ) # Recommendations @@ -539,5 +544,5 @@ def aggregate_results( ai_calls_made=ai_calls, requires_human_review=needs_review, critical_issues=critical_issues, - recommendations=recommendations + recommendations=recommendations, ).model_dump() diff --git a/examples/python_agent_nodes/rag_evaluation/reasoners/relevance.py b/examples/python_agent_nodes/rag_evaluation/reasoners/relevance.py index cd7af7e67..0311ad1b4 100644 --- a/examples/python_agent_nodes/rag_evaluation/reasoners/relevance.py +++ b/examples/python_agent_nodes/rag_evaluation/reasoners/relevance.py @@ -29,10 +29,7 @@ def register_relevance_reasoners(router): # ============================================ @router.reasoner() - async def analyze_question( - question: str, - model: Optional[str] = None - ) -> dict: + async def analyze_question(question: str, model: Optional[str] = None) -> dict: """ Question analyst: Extract intent and requirements. @@ -63,12 +60,14 @@ async def analyze_question( - procedure: step-by-step instructions - opinion: subjective assessment""", schema=QuestionIntent, - model=model + model=model, ) # Include the identified intent - router.note(f"Question intent: \"{result.primary_intent[:70]}...\"", - tags=["relevance", "analysis"]) + router.note( + f'Question intent: "{result.primary_intent[:70]}..."', + tags=["relevance", "analysis"], + ) return result.model_dump() @@ -81,7 +80,7 @@ async def vote_literal_relevance( question: str, response: str, question_analysis: Dict, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Literal juror: Does response literally answer the question? @@ -106,8 +105,8 @@ async def vote_literal_relevance( {response} QUESTION ANALYSIS: -Expected type: {question_analysis.get('expected_type', 'unknown')} -Sub-questions: {question_analysis.get('sub_questions', [])} +Expected type: {question_analysis.get("expected_type", "unknown")} +Sub-questions: {question_analysis.get("sub_questions", [])} Evaluate LITERALLY: - Does the response address the exact words/terms in the question? @@ -120,13 +119,15 @@ async def vote_literal_relevance( - reasoning: Why this score? - confidence: 0.0-1.0 how confident in your assessment""", schema=JurorVote, - model=model + model=model, ) # Include reasoning snippet reason = result.reasoning[:60] if result.reasoning else "Evaluated" - router.note(f"Literal juror ({result.score:.0%}): {reason}...", - tags=["relevance", "jury", "literal"]) + router.note( + f"Literal juror ({result.score:.0%}): {reason}...", + tags=["relevance", "jury", "literal"], + ) return result.model_dump() @@ -139,7 +140,7 @@ async def vote_intent_relevance( question: str, response: str, question_analysis: Dict, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Intent juror: Does response address underlying user need? @@ -164,7 +165,7 @@ async def vote_intent_relevance( {response} PRIMARY INTENT IDENTIFIED: -{question_analysis.get('primary_intent', 'Unknown')} +{question_analysis.get("primary_intent", "Unknown")} Evaluate INTENT: - Does the response help the user achieve their actual goal? @@ -177,13 +178,15 @@ async def vote_intent_relevance( - reasoning: Why this score? - confidence: 0.0-1.0 how confident in your assessment""", schema=JurorVote, - model=model + model=model, ) # Include reasoning snippet reason = result.reasoning[:60] if result.reasoning else "Evaluated" - router.note(f"Intent juror ({result.score:.0%}): {reason}...", - tags=["relevance", "jury", "intent"]) + router.note( + f"Intent juror ({result.score:.0%}): {reason}...", + tags=["relevance", "jury", "intent"], + ) return result.model_dump() @@ -196,7 +199,7 @@ async def vote_scope_relevance( question: str, response: str, question_analysis: Dict, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Scope juror: Is response appropriately scoped? @@ -221,7 +224,7 @@ async def vote_scope_relevance( {response} SUB-QUESTIONS TO ADDRESS: -{question_analysis.get('sub_questions', [])} +{question_analysis.get("sub_questions", [])} Evaluate SCOPE: - Is the response too narrow? (missing important aspects) @@ -235,13 +238,15 @@ async def vote_scope_relevance( - reasoning: Why this score? - confidence: 0.0-1.0 how confident in your assessment""", schema=JurorVote, - model=model + model=model, ) # Include reasoning snippet reason = result.reasoning[:60] if result.reasoning else "Evaluated" - router.note(f"Scope juror ({result.score:.0%}): {reason}...", - tags=["relevance", "jury", "scope"]) + router.note( + f"Scope juror ({result.score:.0%}): {reason}...", + tags=["relevance", "jury", "scope"], + ) return result.model_dump() @@ -256,7 +261,7 @@ async def synthesize_relevance_verdict( literal_vote: Dict, intent_vote: Dict, scope_vote: Dict, - model: Optional[str] = None + model: Optional[str] = None, ) -> dict: """ Jury foreman: Synthesize all juror votes into final verdict. @@ -268,13 +273,15 @@ async def synthesize_relevance_verdict( - Flag high-disagreement cases for human review """ - router.note("Jury foreman synthesizing votes...", tags=["relevance", "synthesis"]) + router.note( + "Jury foreman synthesizing votes...", tags=["relevance", "synthesis"] + ) # Calculate disagreement scores = [ literal_vote.get("score", 0.5), intent_vote.get("score", 0.5), - scope_vote.get("score", 0.5) + scope_vote.get("score", 0.5), ] max_score = max(scores) min_score = min(scores) @@ -288,9 +295,9 @@ async def synthesize_relevance_verdict( total_conf = literal_conf + intent_conf + scope_conf if total_conf > 0: weighted_score = ( - literal_vote.get("score", 0.5) * literal_conf + - intent_vote.get("score", 0.5) * intent_conf + - scope_vote.get("score", 0.5) * scope_conf + literal_vote.get("score", 0.5) * literal_conf + + intent_vote.get("score", 0.5) * intent_conf + + scope_vote.get("score", 0.5) * scope_conf ) / total_conf else: weighted_score = sum(scores) / 3 @@ -303,15 +310,15 @@ async def synthesize_relevance_verdict( RESPONSE: {response} JURY VOTES: -- Literal Juror: {literal_vote.get('score', 0):.2f} - {literal_vote.get('reasoning', 'No reason')} -- Intent Juror: {intent_vote.get('score', 0):.2f} - {intent_vote.get('reasoning', 'No reason')} -- Scope Juror: {scope_vote.get('score', 0):.2f} - {scope_vote.get('reasoning', 'No reason')} +- Literal Juror: {literal_vote.get("score", 0):.2f} - {literal_vote.get("reasoning", "No reason")} +- Intent Juror: {intent_vote.get("score", 0):.2f} - {intent_vote.get("reasoning", "No reason")} +- Scope Juror: {scope_vote.get("score", 0):.2f} - {scope_vote.get("reasoning", "No reason")} DISAGREEMENT LEVEL: {disagreement:.2f} Provide a summary verdict explaining the jury's decision.""", schema=None, # Just get text summary - model=model + model=model, ) verdict = RelevanceVerdict( @@ -320,13 +327,17 @@ async def synthesize_relevance_verdict( intent_score=intent_vote.get("score", 0.5), scope_score=scope_vote.get("score", 0.5), disagreement_level=disagreement, - verdict=str(result) if result else "Jury has reached a verdict." + verdict=str(result) if result else "Jury has reached a verdict.", ) # Include the verdict summary - verdict_snippet = verdict.verdict[:70] if verdict.verdict else "Jury has reached consensus" - router.note(f"Jury verdict ({verdict.overall_score:.0%}): {verdict_snippet}...", - tags=["relevance", "jury", "verdict"]) + verdict_snippet = ( + verdict.verdict[:70] if verdict.verdict else "Jury has reached consensus" + ) + router.note( + f"Jury verdict ({verdict.overall_score:.0%}): {verdict_snippet}...", + tags=["relevance", "jury", "verdict"], + ) return verdict.model_dump() @@ -336,9 +347,7 @@ async def synthesize_relevance_verdict( @router.reasoner() async def evaluate_relevance_full( - question: str, - response: str, - model: Optional[str] = None + question: str, response: str, model: Optional[str] = None ) -> dict: """ Full multi-jury orchestrator for relevance. @@ -352,13 +361,14 @@ async def evaluate_relevance_full( analyze_question -> [literal, intent, scope] -> foreman """ - router.note("Convening jury for relevance assessment...", tags=["relevance", "orchestration", "jury"]) + router.note( + "Convening jury for relevance assessment...", + tags=["relevance", "orchestration", "jury"], + ) # Step 1: Analyze the question question_analysis = await router.app.call( - "rag-evaluation.analyze_question", - question=question, - model=model + "rag-evaluation.analyze_question", question=question, model=model ) router.note("Jury deliberating in parallel...", tags=["relevance", "parallel"]) @@ -369,7 +379,7 @@ async def evaluate_relevance_full( question=question, response=response, question_analysis=question_analysis, - model=model + model=model, ) intent_task = router.app.call( @@ -377,7 +387,7 @@ async def evaluate_relevance_full( question=question, response=response, question_analysis=question_analysis, - model=model + model=model, ) scope_task = router.app.call( @@ -385,7 +395,7 @@ async def evaluate_relevance_full( question=question, response=response, question_analysis=question_analysis, - model=model + model=model, ) # Wait for all jurors @@ -403,11 +413,13 @@ async def evaluate_relevance_full( literal_vote=literal_vote, intent_vote=intent_vote, scope_vote=scope_vote, - model=model + model=model, ) - router.note(f"Jury complete: {verdict['overall_score']:.2f} relevance", - tags=["relevance", "complete"]) + router.note( + f"Jury complete: {verdict['overall_score']:.2f} relevance", + tags=["relevance", "complete"], + ) return verdict @@ -417,9 +429,7 @@ async def evaluate_relevance_full( @router.reasoner() async def evaluate_relevance_quick( - question: str, - response: str, - model: Optional[str] = None + question: str, response: str, model: Optional[str] = None ) -> dict: """ Quick single-reasoner relevance evaluation. @@ -448,11 +458,10 @@ async def evaluate_relevance_quick( - addresses_question: true/false - reasoning: Brief explanation""", schema=QuickRelevance, - model=model + model=model, ) - router.note(f"Quick relevance: {result.score:.2f}", - tags=["relevance", "quick"]) + router.note(f"Quick relevance: {result.score:.2f}", tags=["relevance", "quick"]) # Convert to full verdict format return RelevanceVerdict( @@ -461,5 +470,5 @@ async def evaluate_relevance_quick( intent_score=result.score, scope_score=result.score, disagreement_level=0.0, - verdict=result.reasoning + verdict=result.reasoning, ).model_dump() diff --git a/examples/python_agent_nodes/serverless_hello/main.py b/examples/python_agent_nodes/serverless_hello/main.py index 412e3fd3d..d7b60f309 100644 --- a/examples/python_agent_nodes/serverless_hello/main.py +++ b/examples/python_agent_nodes/serverless_hello/main.py @@ -71,7 +71,9 @@ async def discover(): @api.post("/execute") async def execute(request: Request): payload = await request.json() - result = await asyncio.to_thread(app.handle_serverless, {"path": "/execute", **payload}) + result = await asyncio.to_thread( + app.handle_serverless, {"path": "/execute", **payload} + ) status = result.get("statusCode", 200) body = result.get("body", result) return JSONResponse(content=json.loads(json.dumps(body)), status_code=status) diff --git a/examples/python_agent_nodes/simulation_engine/example.ipynb b/examples/python_agent_nodes/simulation_engine/example.ipynb index 09e5ea63f..ce64c5b33 100644 --- a/examples/python_agent_nodes/simulation_engine/example.ipynb +++ b/examples/python_agent_nodes/simulation_engine/example.ipynb @@ -196,7 +196,7 @@ "PREVIOUS ANALYSIS:\n", "Entity Type: {scenario_analysis.entity_type}\n", "Decision Type: {scenario_analysis.decision_type}\n", - "Possible Decisions: {', '.join(scenario_analysis.decision_options)}\n", + "Possible Decisions: {\", \".join(scenario_analysis.decision_options)}\n", "\n", "Key Insights from Analysis:\n", "{scenario_analysis.analysis}\n", @@ -450,7 +450,7 @@ "{scenario}\n", "\n", "{context_section}AVAILABLE DECISIONS:\n", - "{', '.join(scenario_analysis.decision_options)}\n", + "{\", \".join(scenario_analysis.decision_options)}\n", "\n", "TASK:\n", "Based on who you are, decide how you would respond to this scenario.\n", @@ -646,7 +646,7 @@ "\n", " if total_with_attr > 0:\n", " summary_parts = [\n", - " f\"{val} ({count}/{total_with_attr}, {count*100/total_with_attr:.1f}%)\"\n", + " f\"{val} ({count}/{total_with_attr}, {count * 100 / total_with_attr:.1f}%)\"\n", " for val, count in top_values\n", " ]\n", " attribute_summaries[attr_name] = {\n", @@ -674,7 +674,7 @@ " total_for_attr = sum(patterns.values())\n", " attribute_decision_patterns[attr_name] = (\n", " f\"When {attr_name}={attr_val}: {count}/{total_for_attr} chose '{decision_type}' \"\n", - " f\"({count*100/total_for_attr:.1f}%)\"\n", + " f\"({count * 100 / total_for_attr:.1f}%)\"\n", " )\n", "\n", " # 3. Sample representative examples intelligently\n", @@ -761,7 +761,7 @@ "{scenario}\n", "\n", "{context_block}ATTRIBUTES TRACKED:\n", - "{', '.join(factor_graph.attributes.keys())}\n", + "{\", \".join(factor_graph.attributes.keys())}\n", "\n", "OUTCOME DISTRIBUTION (from {total} entities):\n", "{json.dumps(outcome_dist, indent=2)}\n", @@ -905,7 +905,7 @@ " print(f\"\\n🎯 KEY INSIGHT: {insights.key_insight}\")\n", " print(\"\\n📈 OUTCOME DISTRIBUTION:\")\n", " for decision, pct in insights.outcome_distribution.items():\n", - " print(f\" {decision}: {pct*100:.1f}%\")\n", + " print(f\" {decision}: {pct * 100:.1f}%\")\n", "\n", " return {\n", " \"scenario\": scenario,\n", @@ -1352,7 +1352,7 @@ "print(\"OUTCOME DISTRIBUTION:\")\n", "print(\"=\" * 60)\n", "for decision, pct in insights.outcome_distribution.items():\n", - " print(f\" {decision}: {pct*100:.1f}%\")\n", + " print(f\" {decision}: {pct * 100:.1f}%\")\n", "\n", "print(\"\\n\" + \"=\" * 60)\n", "print(\"DETAILED ANALYSIS:\")\n", diff --git a/examples/python_agent_nodes/simulation_engine/routers/aggregation.py b/examples/python_agent_nodes/simulation_engine/routers/aggregation.py index 90bc68108..caa0df02e 100644 --- a/examples/python_agent_nodes/simulation_engine/routers/aggregation.py +++ b/examples/python_agent_nodes/simulation_engine/routers/aggregation.py @@ -70,7 +70,7 @@ async def aggregate_and_analyze( if total_with_attr > 0: summary_parts = [ - f"{val} ({count}/{total_with_attr}, {count*100/total_with_attr:.1f}%)" + f"{val} ({count}/{total_with_attr}, {count * 100 / total_with_attr:.1f}%)" for val, count in top_values ] attribute_summaries[attr_name] = { @@ -98,7 +98,7 @@ async def aggregate_and_analyze( total_for_attr = sum(patterns.values()) attribute_decision_patterns[attr_name] = ( f"When {attr_name}={attr_val}: {count}/{total_for_attr} chose '{decision_type}' " - f"({count*100/total_for_attr:.1f}%)" + f"({count * 100 / total_for_attr:.1f}%)" ) # 3. Sample representative examples intelligently @@ -185,7 +185,7 @@ async def aggregate_and_analyze( {scenario} {context_block}ATTRIBUTES TRACKED: -{', '.join(factor_graph.attributes.keys())} +{", ".join(factor_graph.attributes.keys())} OUTCOME DISTRIBUTION (from {total} entities): {json.dumps(outcome_dist, indent=2)} diff --git a/examples/python_agent_nodes/simulation_engine/routers/decision.py b/examples/python_agent_nodes/simulation_engine/routers/decision.py index 7a823c72a..d72941121 100644 --- a/examples/python_agent_nodes/simulation_engine/routers/decision.py +++ b/examples/python_agent_nodes/simulation_engine/routers/decision.py @@ -53,7 +53,7 @@ async def simulate_entity_decision( {scenario} {context_section}AVAILABLE DECISIONS: -{', '.join(scenario_analysis.decision_options)} +{", ".join(scenario_analysis.decision_options)} TASK: Based on who you are, decide how you would respond to this scenario. diff --git a/examples/python_agent_nodes/simulation_engine/routers/scenario.py b/examples/python_agent_nodes/simulation_engine/routers/scenario.py index 7fac7f048..db5ba53ac 100644 --- a/examples/python_agent_nodes/simulation_engine/routers/scenario.py +++ b/examples/python_agent_nodes/simulation_engine/routers/scenario.py @@ -115,7 +115,7 @@ async def generate_factor_graph( PREVIOUS ANALYSIS: Entity Type: {scenario_analysis.entity_type} Decision Type: {scenario_analysis.decision_type} -Possible Decisions: {', '.join(scenario_analysis.decision_options)} +Possible Decisions: {", ".join(scenario_analysis.decision_options)} Key Insights from Analysis: {scenario_analysis.analysis} diff --git a/examples/python_agent_nodes/simulation_engine/routers/simulation.py b/examples/python_agent_nodes/simulation_engine/routers/simulation.py index 3becbb4ce..cf5dee27b 100644 --- a/examples/python_agent_nodes/simulation_engine/routers/simulation.py +++ b/examples/python_agent_nodes/simulation_engine/routers/simulation.py @@ -93,7 +93,7 @@ async def run_simulation( print(f"\n🎯 KEY INSIGHT: {insights.key_insight}") print("\n📈 OUTCOME DISTRIBUTION:") for decision, pct in insights.outcome_distribution.items(): - print(f" {decision}: {pct*100:.1f}%") + print(f" {decision}: {pct * 100:.1f}%") return SimulationResult( scenario=scenario, diff --git a/examples/python_agent_nodes/tool_calling/worker.py b/examples/python_agent_nodes/tool_calling/worker.py index 4d875ad98..77364633a 100644 --- a/examples/python_agent_nodes/tool_calling/worker.py +++ b/examples/python_agent_nodes/tool_calling/worker.py @@ -52,7 +52,9 @@ def calculate(operation: str, a: float, b: float) -> dict: } result = ops.get(operation.lower()) if result is None: - return {"error": f"Unknown operation: {operation}. Use: add, subtract, multiply, divide"} + return { + "error": f"Unknown operation: {operation}. Use: add, subtract, multiply, divide" + } return {"operation": operation, "a": a, "b": b, "result": result} diff --git a/examples/python_agent_nodes/voice_dictation/main.py b/examples/python_agent_nodes/voice_dictation/main.py index 2a93a4b2f..73b9f5bba 100644 --- a/examples/python_agent_nodes/voice_dictation/main.py +++ b/examples/python_agent_nodes/voice_dictation/main.py @@ -75,7 +75,9 @@ async def voice(session): will become the live loop once #664 lands. """ turn = await session.input() - result = await session.call("voice-dictation.save_note", text=turn.transcript or turn.text or "") + result = await session.call( + "voice-dictation.save_note", text=turn.transcript or turn.text or "" + ) await session.say(f"Saved {result['chars']} characters.") diff --git a/examples/triggers-demo/agent.py b/examples/triggers-demo/agent.py index 4531c5021..bef1487d4 100644 --- a/examples/triggers-demo/agent.py +++ b/examples/triggers-demo/agent.py @@ -180,12 +180,7 @@ async def summarize_issue(event: dict, trigger: TriggerContext | None = None): "reproduction steps, and (c) what the reporter expects. Plain " "prose, no headers." ), - user=( - f"Repo: {repo}\n" - f"Author: {author}\n" - f"Title: {title}\n\n" - f"Body:\n{body}" - ), + user=(f"Repo: {repo}\nAuthor: {author}\nTitle: {title}\n\nBody:\n{body}"), model="openrouter/anthropic/claude-haiku-4-5", max_tokens=300, ) @@ -203,7 +198,10 @@ async def summarize_issue(event: dict, trigger: TriggerContext | None = None): } if repo and number: await app.memory.set(key=f"issue:{repo}#{number}", data=record) - print(f"[summarize_issue] saved {repo}#{number} — {record['summary'][:120]}…", flush=True) + print( + f"[summarize_issue] saved {repo}#{number} — {record['summary'][:120]}…", + flush=True, + ) return record @@ -233,8 +231,13 @@ async def handle_inbound(payload, trigger: TriggerContext | None = None): "payload": payload, } if trigger and trigger.event_id: - await app.memory.set(key=f"inbound:{trigger.source}:{trigger.event_id}", data=record) - print(f"[handle_inbound] {trigger.source if trigger else 'direct'} event_type={record['event_type']}", flush=True) + await app.memory.set( + key=f"inbound:{trigger.source}:{trigger.event_id}", data=record + ) + print( + f"[handle_inbound] {trigger.source if trigger else 'direct'} event_type={record['event_type']}", + flush=True, + ) return record diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 22cf5f8a0..dc1bec97c 100755 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -173,9 +173,7 @@ def _taken_prerelease_counters( return counters -def _target_base_for_lookup( - current: SemVer, label: str, component: str -) -> SemVer: +def _target_base_for_lookup(current: SemVer, label: str, component: str) -> SemVer: """Replicate the base-selection rule in ``determine_next_version``.""" current_label, _ = current.prerelease_parts() if current.prerelease and current_label == label: @@ -321,11 +319,7 @@ def main(argv: Optional[list[str]] = None) -> int: else: current = SemVer.parse(VERSION_FILE.read_text(encoding="utf-8").strip()) taken: tuple[int, ...] = () - if ( - args.channel == "prerelease" - and args.label - and not args.skip_tag_check - ): + if args.channel == "prerelease" and args.label and not args.skip_tag_check: target_base = _target_base_for_lookup(current, args.label, args.component) taken = tuple( _taken_prerelease_counters(target_base, args.label, _remote_tag_names()) diff --git a/scripts/coverage-aggregate.py b/scripts/coverage-aggregate.py index 0e082ca1b..3cba669e9 100755 --- a/scripts/coverage-aggregate.py +++ b/scripts/coverage-aggregate.py @@ -48,20 +48,25 @@ # when surface sizes shift materially. _SURFACE_WEIGHTS: dict[str, int] = { "control-plane": 24326, - "sdk-go": 1, - "sdk-python": 1, + "sdk-go": 1, + "sdk-python": 1, "sdk-typescript": 1, - "web-ui": 41693, + "web-ui": 41693, } def _badge_color(pct: float) -> str: """Shields.io-style threshold colors.""" - if pct >= 90: return "brightgreen" - if pct >= 80: return "green" - if pct >= 70: return "yellowgreen" - if pct >= 60: return "yellow" - if pct >= 50: return "orange" + if pct >= 90: + return "brightgreen" + if pct >= 80: + return "green" + if pct >= 70: + return "yellowgreen" + if pct >= 60: + return "yellow" + if pct >= 50: + return "orange" return "red" @@ -76,9 +81,9 @@ def _read_json(path: Path) -> dict[str, Any]: def aggregate(report_dir: Path) -> dict[str, Any]: cp_total = _read_total(report_dir / "control-plane.total.txt") sg_total = _read_total(report_dir / "sdk-go.total.txt") - py_data = _read_json(report_dir / "sdk-python-coverage.json") - ts_data = _read_json(report_dir / "sdk-typescript-coverage-summary.json") - ui_data = _read_json(report_dir / "web-ui-coverage-summary.json") + py_data = _read_json(report_dir / "sdk-python-coverage.json") + ts_data = _read_json(report_dir / "sdk-typescript-coverage-summary.json") + ui_data = _read_json(report_dir / "web-ui-coverage-summary.json") surfaces = [ { @@ -149,7 +154,9 @@ def aggregate(report_dir: Path) -> dict[str, Any]: def write_outputs(report_dir: Path, summary: dict[str, Any]) -> None: (report_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") - (report_dir / "badge.json").write_text(json.dumps(summary["badge"], indent=2) + "\n") + (report_dir / "badge.json").write_text( + json.dumps(summary["badge"], indent=2) + "\n" + ) agg = summary["aggregate"]["coverage_percent"] lines = [ @@ -161,9 +168,7 @@ def write_outputs(report_dir: Path, summary: dict[str, Any]) -> None: "| --- | ---: | --- |", ] for s in summary["surfaces"]: - lines.append( - f"| {s['name']} | {s['coverage_percent']:.2f}% | {s['notes']} |" - ) + lines.append(f"| {s['name']} | {s['coverage_percent']:.2f}% | {s['notes']} |") lines.extend( [ "", diff --git a/scripts/coverage-gate.py b/scripts/coverage-gate.py index c461c900e..343c592a9 100755 --- a/scripts/coverage-gate.py +++ b/scripts/coverage-gate.py @@ -57,6 +57,7 @@ # --- data helpers ----------------------------------------------------------- + def _load_json(path: Path) -> dict[str, Any]: try: return json.loads(path.read_text()) @@ -90,11 +91,11 @@ def _surface_map(summary: dict[str, Any]) -> dict[str, float]: # --- rendering helpers ------------------------------------------------------ _GLYPHS = { - "ok": "🟢", - "warn": "🟡", - "problem": "🟠", - "bad": "🔴", - "missing": "⚠️ ", + "ok": "🟢", + "warn": "🟡", + "problem": "🟠", + "bad": "🔴", + "missing": "⚠️ ", } @@ -132,8 +133,7 @@ def _delta(cur: float, base: float | None) -> str: "go tool cover -func=cover.out | sort -k3 -n | head -30" ), "sdk-python": ( - "cd sdk/python && " - "python3 -m pytest --cov=agentfield --cov-report=term-missing" + "cd sdk/python && python3 -m pytest --cov=agentfield --cov-report=term-missing" ), "sdk-typescript": ( "cd sdk/typescript && " @@ -154,31 +154,36 @@ def _reproduce(name: str) -> str: # --- main ------------------------------------------------------------------- + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Run the AgentField coverage gate.", formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) parser.add_argument("--baseline", type=Path, required=True) - parser.add_argument("--config", type=Path, default=REPO_ROOT / ".coverage-gate.toml") - parser.add_argument("--soft", action="store_true", help="report violations but exit 0") + parser.add_argument( + "--config", type=Path, default=REPO_ROOT / ".coverage-gate.toml" + ) + parser.add_argument( + "--soft", action="store_true", help="report violations but exit 0" + ) args = parser.parse_args(argv) - summary = _load_json(args.summary) + summary = _load_json(args.summary) baseline = _load_json(args.baseline) - config = _load_toml(args.config) + config = _load_toml(args.config) thresholds = config.get("thresholds", {}) - min_surface = float(thresholds.get("min_surface", 80.0)) - min_aggregate = float(thresholds.get("min_aggregate", 85.0)) - max_surface_drop = float(thresholds.get("max_surface_drop", 1.0)) + min_surface = float(thresholds.get("min_surface", 80.0)) + min_aggregate = float(thresholds.get("min_aggregate", 85.0)) + max_surface_drop = float(thresholds.get("max_surface_drop", 1.0)) max_aggregate_drop = float(thresholds.get("max_aggregate_drop", 0.5)) - surfaces_now = _surface_map(summary) + surfaces_now = _surface_map(summary) surfaces_base = {k: float(v) for k, v in baseline.get("surfaces", {}).items()} - agg_now = float(summary.get("aggregate", {}).get("coverage_percent", 0.0)) - agg_base = float(baseline.get("aggregate", 0.0)) + agg_now = float(summary.get("aggregate", {}).get("coverage_percent", 0.0)) + agg_base = float(baseline.get("aggregate", 0.0)) # --- evaluate rules ----------------------------------------------------- @@ -186,30 +191,34 @@ def main(argv: list[str] | None = None) -> int: for name, pct in sorted(surfaces_now.items()): if pct < min_surface: - violations.append({ - "rule": "min_surface", - "surface": name, - "value": pct, - "threshold": min_surface, - "message": ( - f"`{name}` coverage {pct:.2f}% is below the per-surface floor " - f"of {min_surface:.0f}%." - ), - "reproduce": _reproduce(name), - }) + violations.append( + { + "rule": "min_surface", + "surface": name, + "value": pct, + "threshold": min_surface, + "message": ( + f"`{name}` coverage {pct:.2f}% is below the per-surface floor " + f"of {min_surface:.0f}%." + ), + "reproduce": _reproduce(name), + } + ) if agg_now < min_aggregate: - violations.append({ - "rule": "min_aggregate", - "surface": "aggregate", - "value": agg_now, - "threshold": min_aggregate, - "message": ( - f"weighted aggregate coverage {agg_now:.2f}% is below the floor " - f"of {min_aggregate:.0f}%." - ), - "reproduce": "./scripts/coverage-summary.sh && cat test-reports/coverage/summary.md", - }) + violations.append( + { + "rule": "min_aggregate", + "surface": "aggregate", + "value": agg_now, + "threshold": min_aggregate, + "message": ( + f"weighted aggregate coverage {agg_now:.2f}% is below the floor " + f"of {min_aggregate:.0f}%." + ), + "reproduce": "./scripts/coverage-summary.sh && cat test-reports/coverage/summary.md", + } + ) for name, pct in sorted(surfaces_now.items()): base = surfaces_base.get(name) @@ -217,36 +226,40 @@ def main(argv: list[str] | None = None) -> int: continue drop = base - pct if drop > max_surface_drop: - violations.append({ - "rule": "max_surface_drop", - "surface": name, - "value": pct, - "baseline": base, - "drop": drop, - "threshold": max_surface_drop, - "message": ( - f"`{name}` regressed {drop:.2f} pp against baseline " - f"({base:.2f}% → {pct:.2f}%). Max allowed drop is {max_surface_drop:.1f} pp." - ), - "reproduce": _reproduce(name), - }) + violations.append( + { + "rule": "max_surface_drop", + "surface": name, + "value": pct, + "baseline": base, + "drop": drop, + "threshold": max_surface_drop, + "message": ( + f"`{name}` regressed {drop:.2f} pp against baseline " + f"({base:.2f}% → {pct:.2f}%). Max allowed drop is {max_surface_drop:.1f} pp." + ), + "reproduce": _reproduce(name), + } + ) agg_drop = agg_base - agg_now if agg_drop > max_aggregate_drop: - violations.append({ - "rule": "max_aggregate_drop", - "surface": "aggregate", - "value": agg_now, - "baseline": agg_base, - "drop": agg_drop, - "threshold": max_aggregate_drop, - "message": ( - f"aggregate regressed {agg_drop:.2f} pp against baseline " - f"({agg_base:.2f}% → {agg_now:.2f}%). Max allowed aggregate drop " - f"is {max_aggregate_drop:.2f} pp." - ), - "reproduce": "./scripts/coverage-summary.sh && cat test-reports/coverage/summary.md", - }) + violations.append( + { + "rule": "max_aggregate_drop", + "surface": "aggregate", + "value": agg_now, + "baseline": agg_base, + "drop": agg_drop, + "threshold": max_aggregate_drop, + "message": ( + f"aggregate regressed {agg_drop:.2f} pp against baseline " + f"({agg_base:.2f}% → {agg_now:.2f}%). Max allowed aggregate drop " + f"is {max_aggregate_drop:.2f} pp." + ), + "reproduce": "./scripts/coverage-summary.sh && cat test-reports/coverage/summary.md", + } + ) passed = not violations @@ -272,7 +285,9 @@ def main(argv: list[str] | None = None) -> int: "name": name, "current": pct, "baseline": surfaces_base.get(name), - "delta_pp": round(pct - surfaces_base[name], 4) if name in surfaces_base else None, + "delta_pp": round(pct - surfaces_base[name], 4) + if name in surfaces_base + else None, "reproduce": _reproduce(name), } for name, pct in sorted(surfaces_now.items()) @@ -298,7 +313,7 @@ def main(argv: list[str] | None = None) -> int: report_cfg = config.get("report", {}) out_dir = args.summary.parent json_path = out_dir / report_cfg.get("json", "gate-status.json") - md_path = out_dir / report_cfg.get("markdown", "gate-report.md") + md_path = out_dir / report_cfg.get("markdown", "gate-report.md") json_path.write_text(json.dumps(status, indent=2) + "\n") @@ -321,7 +336,9 @@ def main(argv: list[str] | None = None) -> int: cur = surfaces_now.get(name) base = surfaces_base.get(name) if cur is None: - md.append(f"| `{name}` | _missing_ | {base:.2f}% | — | {_GLYPHS['missing']} |") + md.append( + f"| `{name}` | _missing_ | {base:.2f}% | — | {_GLYPHS['missing']} |" + ) continue md.append( f"| `{name}` | {cur:.2f}% | " @@ -337,11 +354,15 @@ def main(argv: list[str] | None = None) -> int: if passed: md.append("### ✅ Gate passed") md.append("") - md.append("No surface regressed past the allowed threshold and the aggregate stayed above the floor.") + md.append( + "No surface regressed past the allowed threshold and the aggregate stayed above the floor." + ) else: md.append("### ❌ Gate failed") md.append("") - md.append(f"{len(violations)} rule violation{'s' if len(violations) != 1 else ''}:") + md.append( + f"{len(violations)} rule violation{'s' if len(violations) != 1 else ''}:" + ) md.append("") for v in violations: md.append(f"- **{v['rule']}** on `{v['surface']}` — {v['message']}") @@ -350,11 +371,21 @@ def main(argv: list[str] | None = None) -> int: md.append("") md.append("**For AI coding agents** (primary audience of this repo):") md.append("") - md.append("1. Read `test-reports/coverage/gate-status.json`. It is the canonical, machine-readable source of truth for this gate.") - md.append("2. For every entry in `violations[]`, run its `reproduce` shell command to see the lowest-coverage files on that surface.") - md.append("3. Add tests for the uncovered code paths **in this same PR**. New tests must exercise real code; don't add no-op tests that only import the module.") - md.append("4. Re-run `./scripts/coverage-summary.sh && ./scripts/coverage-gate.py --summary test-reports/coverage/summary.json --baseline coverage-baseline.json` locally before pushing.") - md.append("5. **Do not** lower values in `.coverage-gate.toml` or `coverage-baseline.json` to silence the gate. Those files are the contract between this repo and every agent working in it. Only update them in a PR whose explicit purpose is to move the floor, with the reason documented in the PR description.") + md.append( + "1. Read `test-reports/coverage/gate-status.json`. It is the canonical, machine-readable source of truth for this gate." + ) + md.append( + "2. For every entry in `violations[]`, run its `reproduce` shell command to see the lowest-coverage files on that surface." + ) + md.append( + "3. Add tests for the uncovered code paths **in this same PR**. New tests must exercise real code; don't add no-op tests that only import the module." + ) + md.append( + "4. Re-run `./scripts/coverage-summary.sh && ./scripts/coverage-gate.py --summary test-reports/coverage/summary.json --baseline coverage-baseline.json` locally before pushing." + ) + md.append( + "5. **Do not** lower values in `.coverage-gate.toml` or `coverage-baseline.json` to silence the gate. Those files are the contract between this repo and every agent working in it. Only update them in a PR whose explicit purpose is to move the floor, with the reason documented in the PR description." + ) md.append("") md.append("**Reproduce commands by surface:**") md.append("") diff --git a/scripts/tests/test_bump_version.py b/scripts/tests/test_bump_version.py index 4b850496e..339a71ca8 100644 --- a/scripts/tests/test_bump_version.py +++ b/scripts/tests/test_bump_version.py @@ -31,9 +31,7 @@ def test_stable_from_stable_bumps_patch(): def test_stable_from_prerelease_drops_prerelease(): assert ( str( - determine_next_version( - SemVer.parse("0.1.92-rc.3"), "stable", "patch", None - ) + determine_next_version(SemVer.parse("0.1.92-rc.3"), "stable", "patch", None) ) == "0.1.92" ) @@ -41,9 +39,7 @@ def test_stable_from_prerelease_drops_prerelease(): def test_prerelease_first_rc_from_stable_current(): assert ( - str( - determine_next_version(SemVer.parse("0.1.91"), "prerelease", "patch", "rc") - ) + str(determine_next_version(SemVer.parse("0.1.91"), "prerelease", "patch", "rc")) == "0.1.92-rc.1" ) @@ -98,21 +94,18 @@ def test_taken_counters_parser_filters_unrelated_tags(): def test_target_base_lookup_uses_current_base_when_label_matches(): - assert ( - _target_base_for_lookup(SemVer.parse("0.1.92-rc.5"), "rc", "patch") - == SemVer.parse("0.1.92") - ) + assert _target_base_for_lookup( + SemVer.parse("0.1.92-rc.5"), "rc", "patch" + ) == SemVer.parse("0.1.92") def test_target_base_lookup_bumps_component_when_label_differs(): - assert ( - _target_base_for_lookup(SemVer.parse("0.1.91"), "rc", "patch") - == SemVer.parse("0.1.92") - ) - assert ( - _target_base_for_lookup(SemVer.parse("0.1.92-beta.3"), "rc", "patch") - == SemVer.parse("0.1.93") - ) + assert _target_base_for_lookup( + SemVer.parse("0.1.91"), "rc", "patch" + ) == SemVer.parse("0.1.92") + assert _target_base_for_lookup( + SemVer.parse("0.1.92-beta.3"), "rc", "patch" + ) == SemVer.parse("0.1.93") def test_prerelease_channel_requires_label(): diff --git a/scripts/update_changelog.py b/scripts/update_changelog.py index 50d504e5b..082838505 100755 --- a/scripts/update_changelog.py +++ b/scripts/update_changelog.py @@ -36,9 +36,7 @@ def run_git_cliff(version: str) -> str: ) if result.returncode != 0: msg = result.stderr.strip() or result.stdout.strip() - raise SystemExit( - f"git-cliff failed with exit code {result.returncode}: {msg}" - ) + raise SystemExit(f"git-cliff failed with exit code {result.returncode}: {msg}") return result.stdout.strip() diff --git a/tests/functional/agents/call_chain_agents.py b/tests/functional/agents/call_chain_agents.py index ef476d720..6a9a04904 100644 --- a/tests/functional/agents/call_chain_agents.py +++ b/tests/functional/agents/call_chain_agents.py @@ -41,7 +41,8 @@ def create_worker_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent(node_id=resolved_node_id, **agent_kwargs) @@ -70,7 +71,8 @@ def create_orchestrator_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent(node_id=resolved_node_id, **agent_kwargs) diff --git a/tests/functional/agents/docs_quick_start_agent.py b/tests/functional/agents/docs_quick_start_agent.py index 80bcb6e1d..e1c3bb513 100644 --- a/tests/functional/agents/docs_quick_start_agent.py +++ b/tests/functional/agents/docs_quick_start_agent.py @@ -39,7 +39,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent_kwargs.setdefault("version", "1.0.0") diff --git a/tests/functional/agents/memory_agent.py b/tests/functional/agents/memory_agent.py index 037b6ad1e..e41a78790 100644 --- a/tests/functional/agents/memory_agent.py +++ b/tests/functional/agents/memory_agent.py @@ -33,7 +33,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent( diff --git a/tests/functional/agents/memory_events_agent.py b/tests/functional/agents/memory_events_agent.py index 19db79b6e..638a5730c 100644 --- a/tests/functional/agents/memory_events_agent.py +++ b/tests/functional/agents/memory_events_agent.py @@ -39,7 +39,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent( @@ -54,7 +55,9 @@ async def _append_event(record: Dict[str, Any]) -> None: async with agent._event_lock: agent._captured_events.append(record) - async def _wait_for_event(scope_id: str, key: str, action: str, timeout: float = 8.0): + async def _wait_for_event( + scope_id: str, key: str, action: str, timeout: float = 8.0 + ): loop = asyncio.get_running_loop() deadline = loop.time() + timeout cursor = 0 @@ -91,11 +94,10 @@ async def capture_session_preferences(event): "timestamp": event.timestamp, } await _append_event(record) + if not agent.memory_event_client: raise RuntimeError("Memory event client is not initialized") - agent.memory_event_client.subscribe( - ["preferences.*"], capture_session_preferences - ) + agent.memory_event_client.subscribe(["preferences.*"], capture_session_preferences) def _ensure_execution_context( execution_context: Optional[ExecutionContext], session_id: str, actor_id: str @@ -119,7 +121,9 @@ async def record_session_preference( await scoped_memory.set("preferences.favorite_color", preference) try: - event = await _wait_for_event(session_id, "preferences.favorite_color", "set") + event = await _wait_for_event( + session_id, "preferences.favorite_color", "set" + ) except asyncio.TimeoutError as exc: # pragma: no cover - should not happen raise RuntimeError("Timed out waiting for memory set event") from exc @@ -141,7 +145,9 @@ async def clear_session_preference( await scoped_memory.delete("preferences.favorite_color") try: - event = await _wait_for_event(session_id, "preferences.favorite_color", "delete") + event = await _wait_for_event( + session_id, "preferences.favorite_color", "delete" + ) except asyncio.TimeoutError as exc: # pragma: no cover - should not happen raise RuntimeError("Timed out waiting for memory delete event") from exc @@ -157,7 +163,9 @@ async def get_captured_events() -> Dict[str, Any]: @agent.reasoner(name="get_event_history") async def get_event_history(limit: int = 5) -> Dict[str, Any]: - events = await agent.memory.events.history(patterns="preferences.*", limit=limit) + events = await agent.memory.events.history( + patterns="preferences.*", limit=limit + ) serialized = [event.to_dict() for event in events] return {"history": serialized} diff --git a/tests/functional/agents/memory_events_decorator_agent.py b/tests/functional/agents/memory_events_decorator_agent.py index e60885cba..d1807993a 100644 --- a/tests/functional/agents/memory_events_decorator_agent.py +++ b/tests/functional/agents/memory_events_decorator_agent.py @@ -64,7 +64,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent( @@ -228,9 +229,7 @@ async def fire_multi_pattern(target: str, value: str) -> Dict[str, Any]: if target not in {"first", "second"}: raise ValueError("target must be 'first' or 'second'") - key = ( - MULTI_PATTERN_FIRST_KEY if target == "first" else MULTI_PATTERN_SECOND_KEY - ) + key = MULTI_PATTERN_FIRST_KEY if target == "first" else MULTI_PATTERN_SECOND_KEY cursor = await _event_cursor() await general_session_memory.set(key, value) event = await _wait_for_listener_event( diff --git a/tests/functional/agents/quick_start_agent.py b/tests/functional/agents/quick_start_agent.py index 0e4d4580d..987fb18f2 100644 --- a/tests/functional/agents/quick_start_agent.py +++ b/tests/functional/agents/quick_start_agent.py @@ -41,7 +41,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent( @@ -93,7 +94,9 @@ def create_agent_from_env() -> Agent: Useful if you want to run this module as a standalone script. """ api_key = os.environ["OPENROUTER_API_KEY"] - model = os.environ.get("OPENROUTER_MODEL", "openrouter/google/gemini-2.5-flash-lite") + model = os.environ.get( + "OPENROUTER_MODEL", "openrouter/google/gemini-2.5-flash-lite" + ) node_id = os.environ.get("AGENT_NODE_ID") ai_config = AIConfig( diff --git a/tests/functional/agents/restart_replay_agent.py b/tests/functional/agents/restart_replay_agent.py index bcf039246..4a4ced6a0 100644 --- a/tests/functional/agents/restart_replay_agent.py +++ b/tests/functional/agents/restart_replay_agent.py @@ -69,7 +69,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent_kwargs.setdefault( "async_config", @@ -222,7 +223,9 @@ async def verify_recovery( def create_agent_from_env() -> Agent: api_key = os.environ["OPENROUTER_API_KEY"] - model = os.environ.get("OPENROUTER_MODEL", "openrouter/google/gemini-3.1-flash-lite") + model = os.environ.get( + "OPENROUTER_MODEL", "openrouter/google/gemini-3.1-flash-lite" + ) node_id = os.environ.get("AGENT_NODE_ID") ai_config = AIConfig( diff --git a/tests/functional/agents/router_prefix_agent.py b/tests/functional/agents/router_prefix_agent.py index a3caab8ee..d06125b1d 100644 --- a/tests/functional/agents/router_prefix_agent.py +++ b/tests/functional/agents/router_prefix_agent.py @@ -32,7 +32,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent( diff --git a/tests/functional/agents/scoping_agent.py b/tests/functional/agents/scoping_agent.py index b0dbab540..23c3cbc73 100644 --- a/tests/functional/agents/scoping_agent.py +++ b/tests/functional/agents/scoping_agent.py @@ -31,7 +31,8 @@ def create_agent( agent_kwargs.setdefault("dev_mode", True) agent_kwargs.setdefault("callback_url", callback_url or "http://test-agent") agent_kwargs.setdefault( - "agentfield_server", os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080") + "agentfield_server", + os.environ.get("AGENTFIELD_SERVER", "http://localhost:8080"), ) agent = Agent(node_id=resolved_node_id, **agent_kwargs) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index c4488260a..79646fe88 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -81,6 +81,7 @@ def _get_session_logger() -> FunctionalTestLogger: # Environment and Configuration Fixtures # ============================================================================ + @pytest.fixture(scope="session") def control_plane_url() -> str: """Get the AgentField control plane URL from environment.""" @@ -101,11 +102,13 @@ def openrouter_api_key() -> str: def openrouter_model() -> str: """ Get the OpenRouter model to use for tests from environment. - + IMPORTANT: All tests MUST use this fixture and NOT hardcode model names. This allows us to use cost-effective models for testing. """ - model = os.environ.get("OPENROUTER_MODEL", "openrouter/google/gemini-2.5-flash-lite") + model = os.environ.get( + "OPENROUTER_MODEL", "openrouter/google/gemini-2.5-flash-lite" + ) return model @@ -125,6 +128,7 @@ def test_timeout() -> int: # Control Plane Health Check # ============================================================================ + @pytest.fixture(scope="session") def functional_logger() -> FunctionalTestLogger: """Shared structured logger for the entire functional suite.""" @@ -132,7 +136,9 @@ def functional_logger() -> FunctionalTestLogger: @pytest.fixture(scope="session", autouse=True) -def verify_control_plane(control_plane_url: str, functional_logger: FunctionalTestLogger): +def verify_control_plane( + control_plane_url: str, functional_logger: FunctionalTestLogger +): """Verify that the control plane is accessible before running tests.""" health_url = f"{control_plane_url}/api/v1/health" max_attempts = 30 @@ -143,7 +149,9 @@ def verify_control_plane(control_plane_url: str, functional_logger: FunctionalTe try: response = httpx.get(health_url, timeout=2.0) if response.status_code == 200: - functional_logger.log(f"✓ Control plane is healthy (attempt {attempt + 1})") + functional_logger.log( + f"✓ Control plane is healthy (attempt {attempt + 1})" + ) return except (httpx.RequestError, httpx.TimeoutException): pass @@ -152,7 +160,9 @@ def verify_control_plane(control_plane_url: str, functional_logger: FunctionalTe time.sleep(1) functional_logger.log("Control plane did not respond to health checks in time") - pytest.fail(f"Control plane at {control_plane_url} is not responding to health checks") + pytest.fail( + f"Control plane at {control_plane_url} is not responding to health checks" + ) def _prune_old_logs(directory: Path, retention_seconds: int) -> None: @@ -182,6 +192,7 @@ def _prune_old_logs(directory: Path, retention_seconds: int) -> None: # HTTP Client Fixtures # ============================================================================ + @pytest.fixture async def async_http_client( control_plane_url: str, @@ -209,11 +220,12 @@ async def async_http_client( # AI Configuration Fixtures # ============================================================================ + @pytest.fixture def openrouter_config(openrouter_api_key: str, openrouter_model: str) -> AIConfig: """ Provide an AIConfig configured for OpenRouter. - + IMPORTANT: Uses the OPENROUTER_MODEL environment variable. Default model is cost-effective for testing (gemini-2.5-flash-lite). DO NOT hardcode model names in tests - always use this fixture. @@ -232,50 +244,49 @@ def openrouter_config(openrouter_api_key: str, openrouter_model: str) -> AIConfi # Agent Factory Fixtures # ============================================================================ + @pytest.fixture def make_test_agent(control_plane_url: str) -> Callable[..., Agent]: """ Factory fixture to create test agents. - + Returns a callable that creates and configures agents for testing. Agents are automatically configured to connect to the control plane. - + Usage: def test_example(make_test_agent, openrouter_config): agent = make_test_agent( node_id="test-agent", ai_config=openrouter_config ) - + @agent.reasoner() async def my_reasoner(): return {"status": "ok"} """ created_agents = [] - + def _factory( - node_id: Optional[str] = None, - ai_config: Optional[AIConfig] = None, - **kwargs + node_id: Optional[str] = None, ai_config: Optional[AIConfig] = None, **kwargs ) -> Agent: # Generate unique node ID if not provided if node_id is None: node_id = f"test-agent-{uuid.uuid4().hex[:8]}" - + # Set sensible defaults for testing kwargs.setdefault("agentfield_server", control_plane_url) kwargs.setdefault("dev_mode", True) kwargs.setdefault("callback_url", "http://test-agent") - + if ai_config is not None: kwargs["ai_config"] = ai_config - + agent = Agent(node_id=node_id, **kwargs) created_agents.append(agent) return agent - + yield _factory - + # Cleanup: No explicit cleanup needed as agents are ephemeral in tests @@ -283,34 +294,35 @@ def _factory( async def registered_agent( make_test_agent: Callable, openrouter_config: AIConfig, - async_http_client: httpx.AsyncClient + async_http_client: httpx.AsyncClient, ) -> AsyncGenerator[Agent, None]: """ Provide a test agent that is already registered with the control plane. - + This is a convenience fixture for tests that need a ready-to-use agent. """ import threading import uvicorn - + # Create agent agent = make_test_agent(ai_config=openrouter_config) - + # Add a simple test reasoner @agent.reasoner() async def echo(message: str) -> Dict[str, str]: """Echo back the input message.""" return {"message": message} - + # Find a free port import socket + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind((AGENT_BIND_HOST, 0)) port = s.getsockname()[1] - + # Start agent in background thread agent.base_url = f"http://{AGENT_CALLBACK_HOST}:{port}" - + config = uvicorn.Config( app=agent, host=AGENT_BIND_HOST, @@ -321,25 +333,25 @@ async def echo(message: str) -> Dict[str, str]: ) server = uvicorn.Server(config) loop = asyncio.new_event_loop() - + def run_server(): asyncio.set_event_loop(loop) loop.run_until_complete(server.serve()) - + thread = threading.Thread(target=run_server, daemon=True) thread.start() - + # Wait for agent to be ready await asyncio.sleep(1) - + # Register with control plane try: await agent.agentfield_handler.register_with_agentfield_server(port) agent.agentfield_server = None - + # Wait for registration to complete await asyncio.sleep(1) - + yield agent finally: # Cleanup @@ -353,6 +365,7 @@ def run_server(): # Test Data Fixtures # ============================================================================ + @pytest.fixture def sample_test_input() -> Dict[str, str]: """Provide sample test input data.""" @@ -366,14 +379,13 @@ def sample_test_input() -> Dict[str, str]: # Pytest Configuration # ============================================================================ + def pytest_configure(config): """Configure pytest with custom markers.""" config.addinivalue_line( "markers", "functional: Functional integration tests with real services" ) - config.addinivalue_line( - "markers", "slow: Tests that may take longer to execute" - ) + config.addinivalue_line("markers", "slow: Tests that may take longer to execute") config.addinivalue_line( "markers", "openrouter: Tests that require OpenRouter API access" ) diff --git a/tests/functional/tests/test_af_trigger_cli.py b/tests/functional/tests/test_af_trigger_cli.py index f80d2ee61..ea26669a2 100644 --- a/tests/functional/tests/test_af_trigger_cli.py +++ b/tests/functional/tests/test_af_trigger_cli.py @@ -67,7 +67,9 @@ async def echo(message: str) -> dict: schema = json.loads(stdout) assert "message" in schema.get("properties", {}) - rc, stdout, stderr = await _run_af("ls", "echo", "--node", agent.node_id, "-o", "json") + rc, stdout, stderr = await _run_af( + "ls", "echo", "--node", agent.node_id, "-o", "json" + ) assert rc == 0, stderr listing = json.loads(stdout) assert any( diff --git a/tests/functional/tests/test_ard_discovery.py b/tests/functional/tests/test_ard_discovery.py index 51e1d703f..4f974f5a4 100644 --- a/tests/functional/tests/test_ard_discovery.py +++ b/tests/functional/tests/test_ard_discovery.py @@ -11,7 +11,9 @@ @pytest.mark.functional @pytest.mark.asyncio -async def test_ard_catalog_publish_and_registry_routes(make_test_agent, async_http_client): +async def test_ard_catalog_publish_and_registry_routes( + make_test_agent, async_http_client +): agent = make_test_agent(node_id=unique_node_id("ard-agent")) @agent.reasoner(tags=["legal", "contracts"]) @@ -28,7 +30,8 @@ async def ard_review_contract(text: str) -> dict: publication = next( item for item in dashboard["publications"] - if item["node_id"] == agent.node_id and item["target_id"] == "ard_review_contract" + if item["node_id"] == agent.node_id + and item["target_id"] == "ard_review_contract" ) publication.update( { @@ -58,8 +61,12 @@ async def ard_review_contract(text: str) -> dict: assert catalog["host"]["displayName"] == "AgentField Functional Control Plane" assert catalog["host"]["identifier"] == "did:web:functional.agentfield.local" entries = catalog["entries"] - entry = next(item for item in entries if item["displayName"] == "ARD Contract Reviewer") - assert entry["identifier"].startswith("urn:ai:functional.agentfield.local:agentfield:") + entry = next( + item for item in entries if item["displayName"] == "ARD Contract Reviewer" + ) + assert entry["identifier"].startswith( + "urn:ai:functional.agentfield.local:agentfield:" + ) assert entry["identifier"].endswith(":reasoner:ard_review_contract") assert bool(entry.get("url")) ^ bool(entry.get("data")) assert entry["type"] == "application/openapi+json" @@ -76,12 +83,20 @@ async def ard_review_contract(text: str) -> dict: ) assert search_resp.status_code == 200, search_resp.text search_payload = search_resp.json() - assert any(result["identifier"] == entry["identifier"] for result in search_payload["results"]) + assert any( + result["identifier"] == entry["identifier"] + for result in search_payload["results"] + ) - agents_resp = await async_http_client.get("/api/v1/ard/agents?pageSize=5", timeout=30.0) + agents_resp = await async_http_client.get( + "/api/v1/ard/agents?pageSize=5", timeout=30.0 + ) assert agents_resp.status_code == 200, agents_resp.text agents_payload = agents_resp.json() - assert any(item["identifier"] == entry["identifier"] for item in agents_payload["items"]) + assert any( + item["identifier"] == entry["identifier"] + for item in agents_payload["items"] + ) explore_resp = await async_http_client.post( "/api/v1/ard/explore", @@ -93,5 +108,7 @@ async def ard_review_contract(text: str) -> dict: ) assert explore_resp.status_code == 200, explore_resp.text explore_payload = explore_resp.json() - tag_values = {bucket["value"] for bucket in explore_payload["facets"]["tags"]["buckets"]} + tag_values = { + bucket["value"] for bucket in explore_payload["facets"]["tags"]["buckets"] + } assert "contracts" in tag_values diff --git a/tests/functional/tests/test_go_sdk_cli.py b/tests/functional/tests/test_go_sdk_cli.py index 7d36c965e..a8add46c9 100644 --- a/tests/functional/tests/test_go_sdk_cli.py +++ b/tests/functional/tests/test_go_sdk_cli.py @@ -18,10 +18,14 @@ async def _wait_for_registration(client, node_id: str, timeout: float = 30.0): if resp.status_code == 200: return await asyncio.sleep(1) - raise AssertionError(f"Node {node_id} not registered after {timeout}s (last status={last_status})") + raise AssertionError( + f"Node {node_id} not registered after {timeout}s (last status={last_status})" + ) -async def _wait_for_workflow_run(client, run_id: str, *, expected_reasoners: set[str], timeout: float = 45.0): +async def _wait_for_workflow_run( + client, run_id: str, *, expected_reasoners: set[str], timeout: float = 45.0 +): """Poll workflow run detail endpoint until expected reasoners appear.""" # Give the database a moment to commit the executions await asyncio.sleep(0.5) @@ -47,7 +51,9 @@ async def _wait_for_workflow_run_completed( client, run_id: str, *, expected_reasoners: set[str], timeout: float = 45.0 ): """Poll workflow run detail endpoint until expected reasoners appear and are terminal.""" - timeline = await _wait_for_workflow_run(client, run_id, expected_reasoners=expected_reasoners, timeout=timeout) + timeline = await _wait_for_workflow_run( + client, run_id, expected_reasoners=expected_reasoners, timeout=timeout + ) deadline = time.time() + timeout last_body = None @@ -83,7 +89,9 @@ async def _wait_for_agent_health(url: str, timeout: float = 20.0): raise AssertionError(f"Agent did not become healthy at {url}") -async def _resolve_workflow_id_from_execution(client, execution_id: str, timeout: float = 15.0) -> str: +async def _resolve_workflow_id_from_execution( + client, execution_id: str, timeout: float = 15.0 +) -> str: """Query execution status API to retrieve canonical workflow/run ID.""" deadline = time.time() + timeout last_status = None @@ -92,7 +100,9 @@ async def _resolve_workflow_id_from_execution(client, execution_id: str, timeout last_status = resp.status_code if resp.status_code == 200: data = resp.json() - workflow_id = data.get("workflow_id") or data.get("run_id") or data.get("runId") + workflow_id = ( + data.get("workflow_id") or data.get("run_id") or data.get("runId") + ) if workflow_id: return workflow_id await asyncio.sleep(0.5) @@ -106,8 +116,12 @@ async def _wait_for_execution_logs(client, execution_id: str, timeout: float = 1 deadline = time.time() + timeout last_body = None while time.time() < deadline: - resp = await client.get(f"/api/ui/v1/executions/{execution_id}/logs", params={"tail": "200"}) - assert resp.status_code == 200, f"execution logs failed: {resp.status_code} {resp.text}" + resp = await client.get( + f"/api/ui/v1/executions/{execution_id}/logs", params={"tail": "200"} + ) + assert resp.status_code == 200, ( + f"execution logs failed: {resp.status_code} {resp.text}" + ) body = resp.json() last_body = body entries = body.get("entries", []) @@ -119,7 +133,9 @@ async def _wait_for_execution_logs(client, execution_id: str, timeout: float = 1 ) -async def _wait_for_node_logs_containing(client, node_id: str, marker: str, timeout: float = 15.0): +async def _wait_for_node_logs_containing( + client, node_id: str, marker: str, timeout: float = 15.0 +): """Poll raw node-log proxy until a marker string appears in the visible tail.""" deadline = time.time() + timeout last_joined = "" @@ -129,14 +145,18 @@ async def _wait_for_node_logs_containing(client, node_id: str, marker: str, time params={"tail_lines": "10000"}, timeout=30.0, ) - assert resp.status_code == 200, f"node logs failed: {resp.status_code} {resp.text}" + assert resp.status_code == 200, ( + f"node logs failed: {resp.status_code} {resp.text}" + ) lines = [ln for ln in resp.text.strip().split("\n") if ln.strip()] if lines: last_joined = "\n".join(lines) if marker in last_joined: return lines await asyncio.sleep(0.5) - raise AssertionError(f"Node {node_id} logs never contained marker {marker!r}: {last_joined}") + raise AssertionError( + f"Node {node_id} logs never contained marker {marker!r}: {last_joined}" + ) @pytest.mark.functional @@ -154,7 +174,9 @@ async def test_go_sdk_cli_and_control_plane(async_http_client, control_plane_url try: binary = get_go_agent_binary("hello") except FileNotFoundError: - pytest.skip("Go agent binary not built; ensure go_agents are compiled in test image") + pytest.skip( + "Go agent binary not built; ensure go_agents are compiled in test image" + ) # Start Go agent server pointed at control plane. env_server = { @@ -174,13 +196,17 @@ async def test_go_sdk_cli_and_control_plane(async_http_client, control_plane_url resp = await async_http_client.post( f"/api/v1/execute/{node_id}.demo_echo", json=payload, timeout=30.0 ) - assert resp.status_code == 200, f"execute failed: {resp.status_code} {resp.text}" + assert resp.status_code == 200, ( + f"execute failed: {resp.status_code} {resp.text}" + ) body = resp.json() execution_id = body.get("execution_id") assert execution_id, f"execution_id missing in response: {body}" - workflow_id = await _resolve_workflow_id_from_execution(async_http_client, execution_id) + workflow_id = await _resolve_workflow_id_from_execution( + async_http_client, execution_id + ) # Validate result shape result = body.get("result") or {} assert result.get("name") == "Hello, Agentfield!" @@ -192,15 +218,23 @@ async def test_go_sdk_cli_and_control_plane(async_http_client, control_plane_url expected_reasoners={"demo_echo", "say_hello", "add_emoji"}, timeout=30.0, ) - id_by_reasoner = {node["reasoner_id"]: node["execution_id"] for node in timeline} - parent_by_reasoner = {node["reasoner_id"]: node.get("parent_execution_id") for node in timeline} + id_by_reasoner = { + node["reasoner_id"]: node["execution_id"] for node in timeline + } + parent_by_reasoner = { + node["reasoner_id"]: node.get("parent_execution_id") for node in timeline + } assert parent_by_reasoner.get("demo_echo") in (None, "", "null") assert parent_by_reasoner.get("say_hello") == id_by_reasoner["demo_echo"] assert parent_by_reasoner.get("add_emoji") == id_by_reasoner["say_hello"] # CLI invocation should still work without control plane. - env_cli = {k: v for k, v in os.environ.items() if k not in {"AGENTFIELD_URL", "AGENT_PUBLIC_URL"}} + env_cli = { + k: v + for k, v in os.environ.items() + if k not in {"AGENTFIELD_URL", "AGENT_PUBLIC_URL"} + } env_cli["AGENT_NODE_ID"] = f"{node_id}-cli" cli_proc = await asyncio.create_subprocess_exec( binary, @@ -211,20 +245,26 @@ async def test_go_sdk_cli_and_control_plane(async_http_client, control_plane_url env=env_cli, ) stdout, stderr = await cli_proc.communicate() - assert cli_proc.returncode == 0, f"CLI failed rc={cli_proc.returncode} stderr={stderr.decode()}" + assert cli_proc.returncode == 0, ( + f"CLI failed rc={cli_proc.returncode} stderr={stderr.decode()}" + ) assert "Hello, CLI Functional" in stdout.decode() @pytest.mark.functional @pytest.mark.asyncio -async def test_go_sdk_local_calls_emit_workflow_events(async_http_client, control_plane_url): +async def test_go_sdk_local_calls_emit_workflow_events( + async_http_client, control_plane_url +): """ Ensure Go SDK local composition (CallLocal) still emits workflow events so the control plane builds a full DAG. """ try: get_go_agent_binary("hello") except FileNotFoundError: - pytest.skip("Go agent binary not built; ensure go_agents are compiled in test image") + pytest.skip( + "Go agent binary not built; ensure go_agents are compiled in test image" + ) node_id = unique_node_id("go-local-dag") @@ -244,13 +284,17 @@ async def test_go_sdk_local_calls_emit_workflow_events(async_http_client, contro resp = await async_http_client.post( f"/api/v1/execute/{node_id}.demo_echo", json=payload, timeout=30.0 ) - assert resp.status_code == 200, f"execute failed: {resp.status_code} {resp.text}" + assert resp.status_code == 200, ( + f"execute failed: {resp.status_code} {resp.text}" + ) body = resp.json() execution_id = body.get("execution_id") assert execution_id, f"execution_id missing in response: {body}" - workflow_id = await _resolve_workflow_id_from_execution(async_http_client, execution_id) + workflow_id = await _resolve_workflow_id_from_execution( + async_http_client, execution_id + ) timeline = await _wait_for_workflow_run_completed( async_http_client, workflow_id, @@ -259,8 +303,12 @@ async def test_go_sdk_local_calls_emit_workflow_events(async_http_client, contro ) assert len(timeline) == 3 - id_by_reasoner = {node["reasoner_id"]: node["execution_id"] for node in timeline} - parent_by_reasoner = {node["reasoner_id"]: node.get("parent_execution_id") for node in timeline} + id_by_reasoner = { + node["reasoner_id"]: node["execution_id"] for node in timeline + } + parent_by_reasoner = { + node["reasoner_id"]: node.get("parent_execution_id") for node in timeline + } assert parent_by_reasoner.get("demo_echo") in (None, "", "null") assert parent_by_reasoner.get("say_hello") == id_by_reasoner["demo_echo"] @@ -273,7 +321,9 @@ async def test_go_sdk_local_calls_emit_workflow_events(async_http_client, contro @pytest.mark.functional @pytest.mark.asyncio -async def test_go_sdk_execution_observability_surfaces_structured_and_process_logs(async_http_client, control_plane_url): +async def test_go_sdk_execution_observability_surfaces_structured_and_process_logs( + async_http_client, control_plane_url +): """ Validate execution observability in the canonical functional harness: - structured execution logs are queryable from the control plane for the execution @@ -282,7 +332,9 @@ async def test_go_sdk_execution_observability_surfaces_structured_and_process_lo try: get_go_agent_binary("hello") except FileNotFoundError: - pytest.skip("Go agent binary not built; ensure go_agents are compiled in test image") + pytest.skip( + "Go agent binary not built; ensure go_agents are compiled in test image" + ) node_id = unique_node_id("go-observability") env_server = { @@ -302,13 +354,17 @@ async def test_go_sdk_execution_observability_surfaces_structured_and_process_lo json={"input": {"message": "Execution observability"}}, timeout=30.0, ) - assert resp.status_code == 200, f"execute failed: {resp.status_code} {resp.text}" + assert resp.status_code == 200, ( + f"execute failed: {resp.status_code} {resp.text}" + ) body = resp.json() execution_id = body.get("execution_id") assert execution_id, f"execution_id missing in response: {body}" - workflow_id = await _resolve_workflow_id_from_execution(async_http_client, execution_id) + workflow_id = await _resolve_workflow_id_from_execution( + async_http_client, execution_id + ) entries = await _wait_for_execution_logs(async_http_client, execution_id) assert all(entry.get("execution_id") == execution_id for entry in entries) @@ -319,7 +375,9 @@ async def test_go_sdk_execution_observability_surfaces_structured_and_process_lo assert any(entry.get("system_generated") is True for entry in entries) assert any(entry.get("message") for entry in entries) - proxy_lines = await _wait_for_node_logs_containing(async_http_client, node_id, execution_id) + proxy_lines = await _wait_for_node_logs_containing( + async_http_client, node_id, execution_id + ) parsed = [json.loads(line) for line in proxy_lines] assert parsed, "expected mirrored structured NDJSON lines in raw node logs" assert any(row.get("line", "").find(execution_id) >= 0 for row in parsed) diff --git a/tests/functional/tests/test_hello_world.py b/tests/functional/tests/test_hello_world.py index 589b2bed3..36d2d064f 100644 --- a/tests/functional/tests/test_hello_world.py +++ b/tests/functional/tests/test_hello_world.py @@ -26,7 +26,7 @@ async def test_hello_world_with_openrouter( ): """ Test basic agent execution with real OpenRouter LLM calls. - + This test creates an agent with a simple reasoner that uses OpenRouter to answer a basic math question, then validates the entire execution flow. """ @@ -37,7 +37,7 @@ async def test_hello_world_with_openrouter( node_id=unique_node_id("hello-world-agent"), ai_config=openrouter_config, ) - + # ======================================================================== # Step 2: Define a simple reasoner that uses AI # ======================================================================== @@ -45,10 +45,10 @@ async def test_hello_world_with_openrouter( async def ask_math_question(question: str) -> Dict[str, str]: """ Ask a simple math question and get the answer from the LLM. - + Args: question: The math question to ask - + Returns: Dictionary with the question and answer """ @@ -58,49 +58,47 @@ async def ask_math_question(question: str) -> Dict[str, str]: user=f"Answer this math question with just the number, no explanation: {question}", ) answer_text = getattr(response, "text", None) or str(response) - + return { "question": question, "answer": answer_text.strip(), } - + async with run_agent_server(agent): # ==================================================================== # Step 4: Verify registration via control plane # ==================================================================== nodes_response = await async_http_client.get(f"/api/v1/nodes/{agent.node_id}") - assert nodes_response.status_code == 200, f"Agent not found in registry: {nodes_response.text}" - + assert nodes_response.status_code == 200, ( + f"Agent not found in registry: {nodes_response.text}" + ) + node_data = nodes_response.json() assert node_data["id"] == agent.node_id assert "ask_math_question" in [r["id"] for r in node_data.get("reasoners", [])] - + print(f"✓ Agent registered successfully: {agent.node_id}") - + # ==================================================================== # Step 5: Execute reasoner through control plane # ==================================================================== - execution_request = { - "input": { - "question": "What is 7 + 5?" - } - } - + execution_request = {"input": {"question": "What is 7 + 5?"}} + execution_response = await async_http_client.post( f"/api/v1/reasoners/{agent.node_id}.ask_math_question", json=execution_request, timeout=60.0, ) - + assert execution_response.status_code == 200, ( f"Execution failed: {execution_response.status_code} - {execution_response.text}" ) - + result_data = execution_response.json() - + print("✓ Execution completed successfully") print(f" Response: {result_data}") - + # ==================================================================== # Step 6: Validate execution result # ==================================================================== @@ -108,24 +106,24 @@ async def ask_math_question(question: str) -> Dict[str, str]: assert "result" in result_data, "Missing 'result' in response" assert "node_id" in result_data, "Missing 'node_id' in response" assert "duration_ms" in result_data, "Missing 'duration_ms' in response" - + # Validate node ID assert result_data["node_id"] == agent.node_id - + # Validate result content result = result_data["result"] assert "question" in result assert "answer" in result assert result["question"] == "What is 7 + 5?" - + # The LLM should answer with "12" or something containing "12" answer = result["answer"] assert "12" in answer, f"Expected answer to contain '12', got: {answer}" - + print("✓ Result validation passed") print(f" Question: {result['question']}") print(f" Answer: {result['answer']}") - + # ==================================================================== # Step 7: Validate execution metadata # ==================================================================== @@ -137,18 +135,24 @@ async def ask_math_question(question: str) -> Dict[str, str]: assert "X-Execution-ID" in headers or "x-execution-id" in headers, ( "Missing execution ID in response headers" ) - + # Validate timing assert result_data["duration_ms"] >= 0, "Duration should be non-negative" - + # For OpenRouter calls, we expect some non-trivial execution time - assert result_data["duration_ms"] > 0, "Duration should be greater than 0 for real API calls" - + assert result_data["duration_ms"] > 0, ( + "Duration should be greater than 0 for real API calls" + ) + print("✓ Metadata validation passed") print(f" Duration: {result_data['duration_ms']}ms") - print(f" Workflow ID: {headers.get('X-Workflow-ID', headers.get('x-workflow-id', 'N/A'))}") - print(f" Execution ID: {headers.get('X-Execution-ID', headers.get('x-execution-id', 'N/A'))}") - + print( + f" Workflow ID: {headers.get('X-Workflow-ID', headers.get('x-workflow-id', 'N/A'))}" + ) + print( + f" Execution ID: {headers.get('X-Execution-ID', headers.get('x-execution-id', 'N/A'))}" + ) + print("\n✅ All validations passed! End-to-end test successful.") @@ -157,13 +161,15 @@ async def ask_math_question(question: str) -> Dict[str, str]: async def test_control_plane_health(async_http_client): """ Simple sanity test to ensure the control plane is responding. - + This test doesn't require OpenRouter and serves as a quick smoke test. """ response = await async_http_client.get("/api/v1/health") assert response.status_code == 200 - + health_data = response.json() - assert "status" in health_data or response.text == "OK" or response.status_code == 200 - + assert ( + "status" in health_data or response.text == "OK" or response.status_code == 200 + ) + print("✓ Control plane health check passed") diff --git a/tests/functional/tests/test_memory_events.py b/tests/functional/tests/test_memory_events.py index fc5c10264..322741662 100644 --- a/tests/functional/tests/test_memory_events.py +++ b/tests/functional/tests/test_memory_events.py @@ -33,12 +33,8 @@ async def test_memory_event_listener_captures_session_updates(async_http_client) async with run_agent_server(agent): user_id = unique_node_id("events-user") session_id = f"session::{user_id}" - record_endpoint = ( - f"/api/v1/reasoners/{agent.node_id}.record_session_preference" - ) - clear_endpoint = ( - f"/api/v1/reasoners/{agent.node_id}.clear_session_preference" - ) + record_endpoint = f"/api/v1/reasoners/{agent.node_id}.record_session_preference" + clear_endpoint = f"/api/v1/reasoners/{agent.node_id}.clear_session_preference" captured_endpoint = f"/api/v1/reasoners/{agent.node_id}.get_captured_events" preference = "solarized" @@ -63,9 +59,7 @@ async def test_memory_event_listener_captures_session_updates(async_http_client) assert delete_event["scope_id"] == session_id assert delete_event["previous_data"] == preference - captured = await _invoke_reasoner( - async_http_client, captured_endpoint, {} - ) + captured = await _invoke_reasoner(async_http_client, captured_endpoint, {}) assert len(captured["events"]) >= 2 @@ -81,12 +75,8 @@ async def test_memory_event_history_matches_live_events(async_http_client): async with run_agent_server(agent): user_id = unique_node_id("events-history-user") session_id = f"session::{user_id}" - record_endpoint = ( - f"/api/v1/reasoners/{agent.node_id}.record_session_preference" - ) - clear_endpoint = ( - f"/api/v1/reasoners/{agent.node_id}.clear_session_preference" - ) + record_endpoint = f"/api/v1/reasoners/{agent.node_id}.record_session_preference" + clear_endpoint = f"/api/v1/reasoners/{agent.node_id}.clear_session_preference" history_endpoint = f"/api/v1/reasoners/{agent.node_id}.get_event_history" preference = "amber" @@ -95,9 +85,7 @@ async def test_memory_event_history_matches_live_events(async_http_client): record_endpoint, {"user_id": user_id, "preference": preference}, ) - await _invoke_reasoner( - async_http_client, clear_endpoint, {"user_id": user_id} - ) + await _invoke_reasoner(async_http_client, clear_endpoint, {"user_id": user_id}) # Retry logic to allow event persistence with increased limit relevant_events = [] @@ -120,12 +108,8 @@ async def test_memory_event_history_matches_live_events(async_http_client): f"got {len(relevant_events)}. All events: {history.get('history', [])}" ) - set_event = next( - evt for evt in relevant_events if evt["action"] == "set" - ) - delete_event = next( - evt for evt in relevant_events if evt["action"] == "delete" - ) + set_event = next(evt for evt in relevant_events if evt["action"] == "set") + delete_event = next(evt for evt in relevant_events if evt["action"] == "delete") assert set_event["data"] == preference assert delete_event["previous_data"] == preference diff --git a/tests/functional/tests/test_quick_start.py b/tests/functional/tests/test_quick_start.py index 0f1f8af47..923f5b7ec 100644 --- a/tests/functional/tests/test_quick_start.py +++ b/tests/functional/tests/test_quick_start.py @@ -74,7 +74,9 @@ def log_message(self, *_): return server, thread, f"http://{host}:{port}" -def _start_webhook_capture_server() -> Tuple[ThreadingHTTPServer, threading.Thread, str, "queue.Queue[dict]"]: +def _start_webhook_capture_server() -> Tuple[ + ThreadingHTTPServer, threading.Thread, str, "queue.Queue[dict]" +]: """ Spin up a lightweight HTTP server that captures execution webhooks. """ @@ -122,9 +124,9 @@ async def test_docs_quick_start_demo_echo_flow(async_http_client): assert node_data["id"] == node_id reasoner_ids = [r.get("id") for r in node_data.get("reasoners", [])] - assert any( - rid in {"demo_echo", "echo"} for rid in reasoner_ids - ), f"Reasoner IDs {reasoner_ids} did not include demo_echo/echo" + assert any(rid in {"demo_echo", "echo"} for rid in reasoner_ids), ( + f"Reasoner IDs {reasoner_ids} did not include demo_echo/echo" + ) execution_request = {"input": {"message": DEMO_MESSAGE}} @@ -164,7 +166,9 @@ async def test_docs_quick_start_execution_webhook_contract(async_http_client): try: async with run_agent_server(agent): - webhook_server, webhook_thread, webhook_url, deliveries = _start_webhook_capture_server() + webhook_server, webhook_thread, webhook_url, deliveries = ( + _start_webhook_capture_server() + ) execution_response = await async_http_client.post( f"/api/v1/execute/{node_id}.demo_echo", @@ -211,7 +215,12 @@ async def test_docs_quick_start_execution_webhook_contract(async_http_client): @pytest.mark.functional @pytest.mark.openrouter -@pytest.mark.flaky(reruns=2, reruns_delay=5, condition=True, reason="OpenRouter API can intermittently timeout") +@pytest.mark.flaky( + reruns=2, + reruns_delay=5, + condition=True, + reason="OpenRouter API can intermittently timeout", +) @pytest.mark.asyncio async def test_readme_quick_start_summarize_flow( openrouter_config, @@ -265,7 +274,9 @@ async def test_readme_quick_start_summarize_flow( assert len(summary_text.split()) >= 5, "Summary should contain multiple words" snippet = result.get("content_snippet", "") - assert "Example Domain" in snippet, "Snippet should contain fetched page content" + assert "Example Domain" in snippet, ( + "Snippet should contain fetched page content" + ) assert len(snippet) > 0 assert result_data["duration_ms"] > 0 diff --git a/tests/functional/tests/test_restart_replay.py b/tests/functional/tests/test_restart_replay.py index 0f16b765c..92f5387c0 100644 --- a/tests/functional/tests/test_restart_replay.py +++ b/tests/functional/tests/test_restart_replay.py @@ -8,7 +8,9 @@ from utils import run_agent_server, unique_node_id -async def _poll_execution(client, execution_id: str, *, timeout: float = 180.0) -> Dict[str, Any]: +async def _poll_execution( + client, execution_id: str, *, timeout: float = 180.0 +) -> Dict[str, Any]: deadline = asyncio.get_running_loop().time() + timeout last_payload: Dict[str, Any] = {} @@ -34,7 +36,9 @@ async def _get_lightweight_dag(client, run_id: str) -> Dict[str, Any]: return response.json() -def _find_node(dag: Dict[str, Any], reasoner_id: str, status: str | None = None) -> Dict[str, Any]: +def _find_node( + dag: Dict[str, Any], reasoner_id: str, status: str | None = None +) -> Dict[str, Any]: for node in dag.get("timeline", []): if node.get("reasoner_id") != reasoner_id: continue @@ -94,7 +98,10 @@ async def test_restart_reuses_successful_calls_and_continues_complex_openrouter_ restart_response = await async_http_client.post( f"/api/v1/executions/{failed_node['execution_id']}/restart", - json={"reuse": "succeeded-before", "reason": "functional restart replay test"}, + json={ + "reuse": "succeeded-before", + "reason": "functional restart replay test", + }, timeout=20.0, ) assert restart_response.status_code == 202, restart_response.text @@ -122,7 +129,9 @@ async def test_restart_reuses_successful_calls_and_continues_complex_openrouter_ assert final_counts["synthesize"] == 2 assert final_counts["verify_recovery"] == 1 - restarted_dag = await _get_lightweight_dag(async_http_client, restart_body["run_id"]) + restarted_dag = await _get_lightweight_dag( + async_http_client, restart_body["run_id"] + ) timeline = restarted_dag.get("timeline", []) reused = [ node diff --git a/tests/functional/tests/test_scoping_headers.py b/tests/functional/tests/test_scoping_headers.py index 16086c855..bf019a409 100644 --- a/tests/functional/tests/test_scoping_headers.py +++ b/tests/functional/tests/test_scoping_headers.py @@ -55,7 +55,10 @@ async def test_session_scope_helper_overrides_headers(async_http_client): headers=write_headers, ) assert write_result["scoped_value"] == "session-value" - assert write_result["execution_context"]["session_id"] == write_headers["X-Session-ID"] + assert ( + write_result["execution_context"]["session_id"] + == write_headers["X-Session-ID"] + ) read_headers = { "X-Session-ID": _random_id("incoming-session"), @@ -75,7 +78,10 @@ async def test_session_scope_helper_overrides_headers(async_http_client): assert read_result["scoped_value"] == "session-value" assert read_result["exists"] is True assert key in (read_result["keys"] or []) - assert read_result["execution_context"]["session_id"] == read_headers["X-Session-ID"] + assert ( + read_result["execution_context"]["session_id"] + == read_headers["X-Session-ID"] + ) @pytest.mark.functional @@ -120,7 +126,9 @@ async def test_actor_scope_helper_overrides_headers(async_http_client): ) assert read_result["scoped_value"] == "actor-value" assert key in (read_result["keys"] or []) - assert read_result["execution_context"]["actor_id"] == read_headers["X-Actor-ID"] + assert ( + read_result["execution_context"]["actor_id"] == read_headers["X-Actor-ID"] + ) @pytest.mark.functional @@ -159,4 +167,7 @@ async def test_workflow_scope_helper_overrides_headers(async_http_client): ) assert read_result["scoped_value"] == "workflow-value" assert key in (read_result["keys"] or []) - assert read_result["execution_context"]["workflow_id"] == read_headers["X-Workflow-ID"] + assert ( + read_result["execution_context"]["workflow_id"] + == read_headers["X-Workflow-ID"] + ) diff --git a/tests/functional/tests/test_serverless_agents.py b/tests/functional/tests/test_serverless_agents.py index 755c52ceb..a91d83755 100644 --- a/tests/functional/tests/test_serverless_agents.py +++ b/tests/functional/tests/test_serverless_agents.py @@ -60,7 +60,9 @@ async def _wait_for_port(host: str, port: int, timeout: float = 15.0, process=No raise AssertionError(f"Port {host}:{port} did not open in time: {last_error}") -async def _register_serverless(_async_http_client, invocation_url: str, *, retries: int = 6): +async def _register_serverless( + _async_http_client, invocation_url: str, *, retries: int = 6 +): """ Register a serverless function using the CLI exactly as documented. @@ -97,7 +99,9 @@ async def _register_serverless_via_cli(invocation_url: str): return {"ok": False, "error": "missing-cli", "candidates": candidates} env = os.environ.copy() - env.setdefault("AGENTFIELD_SERVER", env.get("CONTROL_PLANE_URL", "http://localhost:8080")) + env.setdefault( + "AGENTFIELD_SERVER", env.get("CONTROL_PLANE_URL", "http://localhost:8080") + ) token = env.get("AGENTFIELD_TOKEN") cmd = [af_bin, "nodes", "register-serverless", "--url", invocation_url, "--json"] @@ -134,7 +138,9 @@ async def _register_serverless_via_cli(invocation_url: str): @asynccontextmanager -async def run_python_serverless_agent(node_id: str, control_plane_url: str) -> AsyncIterator[str]: +async def run_python_serverless_agent( + node_id: str, control_plane_url: str +) -> AsyncIterator[str]: """ Start a lightweight FastAPI wrapper that delegates to Agent.handle_serverless. """ @@ -159,7 +165,10 @@ async def hello(name: str = "AgentField") -> dict: # type: ignore[return-type] @app.reasoner() async def relay(target: str, message: str = "ping") -> dict: # type: ignore[return-type] downstream = await app.call(target, name=message) - return {"downstream": downstream, "parent_execution_id": getattr(app.ctx, "execution_id", None)} + return { + "downstream": downstream, + "parent_execution_id": getattr(app.ctx, "execution_id", None), + } fastapi_app = FastAPI() @@ -170,7 +179,9 @@ async def discover(): async def _handle(request: Request, override_path: Optional[str] = None): payload = await request.json() path = override_path or payload.get("path") or "/execute" - result = await asyncio.to_thread(app.handle_serverless, {"path": path, **payload}) + result = await asyncio.to_thread( + app.handle_serverless, {"path": path, **payload} + ) status = result.get("statusCode", 200) body = result.get("body", result) return JSONResponse(content=body, status_code=status) @@ -213,7 +224,9 @@ def run_server(): @asynccontextmanager -async def run_ts_serverless_agent(node_id: str, control_plane_url: str) -> AsyncIterator[Tuple[str, asyncio.subprocess.Process]]: +async def run_ts_serverless_agent( + node_id: str, control_plane_url: str +) -> AsyncIterator[Tuple[str, asyncio.subprocess.Process]]: port = _get_free_port() env = os.environ.copy() env.update( @@ -225,7 +238,9 @@ async def run_ts_serverless_agent(node_id: str, control_plane_url: str) -> Async } ) env.setdefault("NODE_PATH", "/usr/local/lib/node_modules:/usr/lib/node_modules") - script_path = Path(__file__).resolve().parent.parent / "ts_agents" / "serverless-agent.mjs" + script_path = ( + Path(__file__).resolve().parent.parent / "ts_agents" / "serverless-agent.mjs" + ) process = await asyncio.create_subprocess_exec( "node", @@ -249,7 +264,9 @@ async def run_ts_serverless_agent(node_id: str, control_plane_url: str) -> Async @asynccontextmanager -async def run_go_serverless_agent(node_id: str, control_plane_url: str) -> AsyncIterator[str]: +async def run_go_serverless_agent( + node_id: str, control_plane_url: str +) -> AsyncIterator[str]: port = _get_free_port() env = { **os.environ, @@ -266,10 +283,14 @@ async def run_go_serverless_agent(node_id: str, control_plane_url: str) -> Async @pytest.mark.functional @pytest.mark.asyncio -async def test_python_serverless_agent_registers_and_executes(async_http_client, control_plane_url): +async def test_python_serverless_agent_registers_and_executes( + async_http_client, control_plane_url +): node_id = unique_node_id("py-svless") - async with run_python_serverless_agent(node_id, control_plane_url) as invocation_url: + async with run_python_serverless_agent( + node_id, control_plane_url + ) as invocation_url: await _register_serverless(async_http_client, invocation_url) resp = await async_http_client.post( @@ -281,7 +302,9 @@ async def test_python_serverless_agent_registers_and_executes(async_http_client, body = resp.json() result = body.get("result", {}) assert result.get("greeting") == "Hello, Lambda!" - assert result.get("execution_id"), "execution_id should propagate to serverless reasoner" + assert result.get("execution_id"), ( + "execution_id should propagate to serverless reasoner" + ) @pytest.mark.functional @@ -293,7 +316,9 @@ async def test_serverless_python_chain_calls(async_http_client, control_plane_ur async with run_python_serverless_agent(child_id, control_plane_url) as child_url: await _register_serverless(async_http_client, child_url) - async with run_python_serverless_agent(parent_id, control_plane_url) as parent_url: + async with run_python_serverless_agent( + parent_id, control_plane_url + ) as parent_url: await _register_serverless(async_http_client, parent_url) resp = await async_http_client.post( @@ -304,7 +329,9 @@ async def test_serverless_python_chain_calls(async_http_client, control_plane_ur assert resp.status_code == 200, resp.text result = resp.json().get("result", {}) assert result.get("downstream", {}).get("greeting") == "Hello, hi-child!" - assert result.get("parent_execution_id"), "parent execution id should be set on relay reasoner" + assert result.get("parent_execution_id"), ( + "parent execution id should be set on relay reasoner" + ) @pytest.mark.functional @@ -312,7 +339,10 @@ async def test_serverless_python_chain_calls(async_http_client, control_plane_ur async def test_typescript_serverless_agent(async_http_client, control_plane_url): node_id = unique_node_id("ts-svless") - async with run_ts_serverless_agent(node_id, control_plane_url) as (invocation_url, process): + async with run_ts_serverless_agent(node_id, control_plane_url) as ( + invocation_url, + process, + ): await _register_serverless(async_http_client, invocation_url) resp = await async_http_client.post( @@ -339,7 +369,10 @@ async def test_typescript_serverless_chain(async_http_client, control_plane_url) child_id = unique_node_id("ts-svless-child") parent_id = unique_node_id("ts-svless-parent") - async with run_ts_serverless_agent(child_id, control_plane_url) as (child_url, child_process): + async with run_ts_serverless_agent(child_id, control_plane_url) as ( + child_url, + child_process, + ): await _register_serverless(async_http_client, child_url) async with run_ts_serverless_agent(parent_id, control_plane_url) as ( @@ -373,7 +406,9 @@ async def test_typescript_serverless_chain(async_http_client, control_plane_url) result = resp.json().get("result", {}) downstream = result.get("downstream", {}) assert downstream.get("greeting") == "Hello, ts-child!" - assert downstream.get("executionId") or downstream.get("execution_id"), "child execution id should propagate" + assert downstream.get("executionId") or downstream.get("execution_id"), ( + "child execution id should propagate" + ) @pytest.mark.functional @@ -409,7 +444,9 @@ async def test_go_serverless_chain(async_http_client, control_plane_url): resp = await async_http_client.post( f"/api/v1/reasoners/{parent_id}.relay", - json={"input": {"target": f"{child_id}.hello", "message": "gopher-child"}}, + json={ + "input": {"target": f"{child_id}.hello", "message": "gopher-child"} + }, timeout=40.0, ) assert resp.status_code == 200, resp.text diff --git a/tests/functional/tests/test_ts_agent.py b/tests/functional/tests/test_ts_agent.py index 76439ee0b..6d1ab7410 100644 --- a/tests/functional/tests/test_ts_agent.py +++ b/tests/functional/tests/test_ts_agent.py @@ -43,7 +43,9 @@ async def run_ts_agent(node_id: str): env.setdefault("NODE_PATH", "/usr/local/lib/node_modules:/usr/lib/node_modules") # Resolve the agent script location relative to the tests/functional root - script_path = os.path.join(os.path.dirname(__file__), "..", "ts_agents", "echo-agent.mjs") + script_path = os.path.join( + os.path.dirname(__file__), "..", "ts_agents", "echo-agent.mjs" + ) process = await asyncio.create_subprocess_exec( "node", @@ -64,7 +66,9 @@ async def run_ts_agent(node_id: str): process.kill() -async def _wait_for_registration(http_client, node_id: str, process, timeout: float = 30.0): +async def _wait_for_registration( + http_client, node_id: str, process, timeout: float = 30.0 +): deadline = asyncio.get_event_loop().time() + timeout last_error = None while asyncio.get_event_loop().time() < deadline: @@ -83,10 +87,14 @@ async def _wait_for_registration(http_client, node_id: str, process, timeout: fl last_error = str(exc) await asyncio.sleep(0.5) - raise AssertionError(f"Node {node_id} did not register in time. Last error: {last_error}") + raise AssertionError( + f"Node {node_id} did not register in time. Last error: {last_error}" + ) -async def _wait_for_port_ready(port: int, process, host: str = "127.0.0.1", timeout: float = 15.0): +async def _wait_for_port_ready( + port: int, process, host: str = "127.0.0.1", timeout: float = 15.0 +): """ Ensure the TS agent has actually bound its HTTP listener before hitting it via the control plane. This avoids a race where the agent registers/heartbeats before its server starts listening. @@ -132,7 +140,10 @@ async def test_typescript_agent_registers_and_executes(async_http_client): # Collect logs if execution fails for easier debugging if resp.status_code != 200: - async def _drain_process(proc: asyncio.subprocess.Process, timeout: float = 5.0): + + async def _drain_process( + proc: asyncio.subprocess.Process, timeout: float = 5.0 + ): """ Capture stdout/stderr without hanging the test if the agent stays alive. We explicitly terminate on timeout so we don't block on a long-lived heartbeat loop. @@ -142,10 +153,14 @@ async def _drain_process(proc: asyncio.subprocess.Process, timeout: float = 5.0) except asyncio.TimeoutError: proc.terminate() try: - return await asyncio.wait_for(proc.communicate(), timeout=timeout) + return await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) except asyncio.TimeoutError: proc.kill() - return await asyncio.wait_for(proc.communicate(), timeout=timeout) + return await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) stdout, stderr = await _drain_process(process) print("TS agent stdout:", stdout.decode("utf-8"), file=sys.stderr) diff --git a/tests/functional/tests/test_ui_node_logs_proxy.py b/tests/functional/tests/test_ui_node_logs_proxy.py index 918bfa984..04e51de81 100644 --- a/tests/functional/tests/test_ui_node_logs_proxy.py +++ b/tests/functional/tests/test_ui_node_logs_proxy.py @@ -49,7 +49,9 @@ async def noisy(): assert lines, "expected NDJSON lines from log proxy" parsed = [json.loads(ln) for ln in lines] - assert all("seq" in row and "stream" in row and "line" in row for row in parsed) + assert all( + "seq" in row and "stream" in row and "line" in row for row in parsed + ) last_joined = "\n".join(lines) if "log-proxy-marker-stdout" in last_joined: diff --git a/tests/functional/tests/test_vc_cli.py b/tests/functional/tests/test_vc_cli.py index c661f89c0..c9a0e8618 100644 --- a/tests/functional/tests/test_vc_cli.py +++ b/tests/functional/tests/test_vc_cli.py @@ -37,7 +37,9 @@ async def _wait_for_vc_chain( await asyncio.sleep(1.0) -async def _run_vc_verify(vc_file: Path, *, expect_success: bool = True) -> Dict[str, Any]: +async def _run_vc_verify( + vc_file: Path, *, expect_success: bool = True +) -> Dict[str, Any]: """Execute `af vc verify` and parse the JSON payload.""" process = await asyncio.create_subprocess_exec( "af", @@ -64,20 +66,22 @@ async def _run_vc_verify(vc_file: Path, *, expect_success: bool = True) -> Dict[ ) from exc if expect_success: - assert ( - process.returncode == 0 - ), f"`af vc verify` failed unexpectedly: rc={process.returncode}, stderr={stderr_text}, stdout={stdout_text}" + assert process.returncode == 0, ( + f"`af vc verify` failed unexpectedly: rc={process.returncode}, stderr={stderr_text}, stdout={stdout_text}" + ) else: - assert ( - process.returncode != 0 - ), "`af vc verify` succeeded unexpectedly when tampering should fail" + assert process.returncode != 0, ( + "`af vc verify` succeeded unexpectedly when tampering should fail" + ) return payload @pytest.mark.functional @pytest.mark.asyncio -async def test_vc_cli_verifies_workflow_chain(make_test_agent, async_http_client, tmp_path): +async def test_vc_cli_verifies_workflow_chain( + make_test_agent, async_http_client, tmp_path +): """ Validate the documentation workflow (docs/core-concepts/identity-and-trust) that exports a workflow VC chain and verifies it offline via `af vc verify`. @@ -112,10 +116,16 @@ async def attest_event(event_type: str, payload: Dict[str, Any]) -> Dict[str, An assert workflow_id, "Workflow ID header missing from response" assert execution_id, "Execution ID header missing from response" - vc_chain = await _wait_for_vc_chain(async_http_client, workflow_id, minimum_components=1) + vc_chain = await _wait_for_vc_chain( + async_http_client, workflow_id, minimum_components=1 + ) - component_execution_ids = {vc["execution_id"] for vc in vc_chain["component_vcs"]} - assert execution_id in component_execution_ids, "Execution VC not found in chain" + component_execution_ids = { + vc["execution_id"] for vc in vc_chain["component_vcs"] + } + assert execution_id in component_execution_ids, ( + "Execution VC not found in chain" + ) vc_file = tmp_path / f"{workflow_id}-vc-chain.json" vc_file.write_text(json.dumps(vc_chain, indent=2)) @@ -133,20 +143,24 @@ async def attest_event(event_type: str, payload: Dict[str, Any]) -> Dict[str, An # Tamper with an execution VC payload and expect verification to fail exec_tampered = copy.deepcopy(vc_chain) - exec_tampered["component_vcs"][0]["vc_document"]["credentialSubject"]["execution"][ - "status" - ] = "tampered" + exec_tampered["component_vcs"][0]["vc_document"]["credentialSubject"][ + "execution" + ]["status"] = "tampered" exec_tampered_file = tmp_path / f"{workflow_id}-vc-chain-exec-tampered.json" exec_tampered_file.write_text(json.dumps(exec_tampered, indent=2)) - tampered_exec_result = await _run_vc_verify(exec_tampered_file, expect_success=False) + tampered_exec_result = await _run_vc_verify( + exec_tampered_file, expect_success=False + ) assert tampered_exec_result["valid"] is False # Tamper with the workflow-level VC document as well workflow_tampered = copy.deepcopy(vc_chain) workflow_doc = workflow_tampered["workflow_vc"]["vc_document"] workflow_doc["credentialSubject"]["status"] = "tampered" - workflow_tampered_file = tmp_path / f"{workflow_id}-vc-chain-workflow-tampered.json" + workflow_tampered_file = ( + tmp_path / f"{workflow_id}-vc-chain-workflow-tampered.json" + ) workflow_tampered_file.write_text(json.dumps(workflow_tampered, indent=2)) tampered_workflow_result = await _run_vc_verify( diff --git a/tests/functional/tests/test_waiting_state.py b/tests/functional/tests/test_waiting_state.py index e9d53380b..cdcc14e8d 100644 --- a/tests/functional/tests/test_waiting_state.py +++ b/tests/functional/tests/test_waiting_state.py @@ -36,7 +36,9 @@ def _unique_node_id(prefix: str = "test-waiting") -> str: # --------------------------------------------------------------------------- -async def _start_execution(client, node_id: str, reasoner_id: str, input_data: dict) -> dict: +async def _start_execution( + client, node_id: str, reasoner_id: str, input_data: dict +) -> dict: """Start a sync execution and return the response payload.""" resp = await client.post( f"/api/v1/execute/{node_id}.{reasoner_id}", @@ -73,7 +75,9 @@ async def _get_approval_status(client, node_id: str, execution_id: str) -> dict: return resp.json(), resp.status_code -async def _send_approval_webhook(client, request_id: str, decision: str, feedback: str = "") -> dict: +async def _send_approval_webhook( + client, request_id: str, decision: str, feedback: str = "" +) -> dict: """Send an approval webhook to resolve a waiting execution.""" body = { "requestId": request_id, @@ -99,7 +103,9 @@ async def _send_approval_webhook(client, request_id: str, decision: str, feedbac def _sign_approval_webhook_payload(payload: dict) -> tuple[bytes, str]: secret = os.environ.get("APPROVAL_WEBHOOK_SECRET", "test-approval-webhook-secret") - body_bytes = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + body_bytes = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode( + "utf-8" + ) sig = hmac.new(secret.encode("utf-8"), body_bytes, hashlib.sha256).hexdigest() return body_bytes, f"sha256={sig}" @@ -122,7 +128,9 @@ async def _get_execution_status(client, execution_id: str) -> dict: @pytest.mark.functional @pytest.mark.asyncio -async def test_approval_request_transitions_to_waiting(make_test_agent, async_http_client): +async def test_approval_request_transitions_to_waiting( + make_test_agent, async_http_client +): """ Verify that requesting approval transitions an execution from running to waiting state. @@ -157,12 +165,16 @@ async def slow_task(message: str) -> dict: ) # Since execution is already succeeded, this should be rejected - assert status_code == 409, f"Expected conflict, got {status_code}: {approval_result}" + assert status_code == 409, ( + f"Expected conflict, got {status_code}: {approval_result}" + ) @pytest.mark.functional @pytest.mark.asyncio -async def test_full_approval_flow_via_async_execution(make_test_agent, async_http_client): +async def test_full_approval_flow_via_async_execution( + make_test_agent, async_http_client +): """ Full end-to-end approval flow using async execution: 1. Start async execution (returns immediately while running) @@ -204,7 +216,9 @@ async def long_running_task(message: str) -> dict: approval_request_url=f"https://hub.example.com/review/{approval_request_id}", expires_in_hours=1, ) - assert status_code == 200, f"Request approval failed ({status_code}): {approval_result}" + assert status_code == 200, ( + f"Request approval failed ({status_code}): {approval_result}" + ) assert approval_result["status"] == "pending" # Poll approval status — should be pending @@ -213,7 +227,10 @@ async def long_running_task(message: str) -> dict: ) assert status_code == 200 assert status_result["status"] == "pending" - assert status_result.get("request_url") == f"https://hub.example.com/review/{approval_request_id}" + assert ( + status_result.get("request_url") + == f"https://hub.example.com/review/{approval_request_id}" + ) # Send webhook to approve webhook_result, webhook_status = await _send_approval_webhook( @@ -222,7 +239,9 @@ async def long_running_task(message: str) -> dict: "approved", feedback="LGTM!", ) - assert webhook_status == 200, f"Webhook failed ({webhook_status}): {webhook_result}" + assert webhook_status == 200, ( + f"Webhook failed ({webhook_status}): {webhook_result}" + ) assert webhook_result["decision"] == "approved" assert webhook_result["new_status"] == "running" @@ -392,7 +411,10 @@ async def dup_task(message: str) -> dict: # First approval request r1, s1 = await _request_approval( - async_http_client, node_id, execution_id, f"req-dup-1-{uuid.uuid4().hex[:8]}" + async_http_client, + node_id, + execution_id, + f"req-dup-1-{uuid.uuid4().hex[:8]}", ) assert s1 == 200 @@ -400,7 +422,10 @@ async def dup_task(message: str) -> dict: # After the first request, execution is in "waiting" state (not "running"), # so the handler rejects with "invalid_state" before reaching the duplicate check. r2, s2 = await _request_approval( - async_http_client, node_id, execution_id, f"req-dup-2-{uuid.uuid4().hex[:8]}" + async_http_client, + node_id, + execution_id, + f"req-dup-2-{uuid.uuid4().hex[:8]}", ) assert s2 == 409 assert r2.get("error") == "invalid_state" diff --git a/tests/functional/utils/logging.py b/tests/functional/utils/logging.py index 04af8d62a..9853a1d6a 100644 --- a/tests/functional/utils/logging.py +++ b/tests/functional/utils/logging.py @@ -103,7 +103,9 @@ def log_request(self, request: httpx.Request) -> Dict[str, Any]: request_id = uuid.uuid4().hex[:8] start = time.perf_counter() - path = request.url.raw_path.decode() if request.url.raw_path else request.url.path + path = ( + request.url.raw_path.decode() if request.url.raw_path else request.url.path + ) self.log(f"[HTTP {request_id}] --> {request.method} {path}") headers = self._filtered_headers(request.headers) @@ -133,7 +135,9 @@ async def log_response( if not streamed: await response.aread() duration_ms = (time.perf_counter() - context["start"]) * 1000.0 - path = request.url.raw_path.decode() if request.url.raw_path else request.url.path + path = ( + request.url.raw_path.decode() if request.url.raw_path else request.url.path + ) self.log( f"[HTTP {context['id']}] <-- {response.status_code} {request.method} {path} " f"({duration_ms:.1f}ms)" @@ -169,7 +173,13 @@ def _filtered_headers(self, headers: httpx.Headers) -> str: rendered = [] for key, value in headers.items(): normalized = key.lower() - if normalized in {"content-length", "user-agent", "accept", "connection", "host"}: + if normalized in { + "content-length", + "user-agent", + "accept", + "connection", + "host", + }: continue if any(token in normalized for token in _SENSITIVE_HEADERS): value = "[redacted]" @@ -181,7 +191,13 @@ def _build_curl_command(self, request: httpx.Request) -> str: for key, value in request.headers.items(): normalized = key.lower() - if normalized in {"content-length", "user-agent", "accept", "connection", "host"}: + if normalized in { + "content-length", + "user-agent", + "accept", + "connection", + "host", + }: continue if any(token in normalized for token in _SENSITIVE_HEADERS): value = "[redacted]"