diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f3101..eca99df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.16] - 2026-02-23 + +### Added + +- System prompt caching in the consensus pipeline for improved performance. +- MCP Client lifecycle monitoring with liveness guards and crash-to-error logging. + +### Fixed + +- Cost display budget timeouts and model attribution loss after child dismissal. +- Infinite continuation loop in fast-path consensus. +- `Decimal.decimal(nil)` crash in `Tracker.over_budget?` for N/A budgets. +- `Response.text()` blindness to pure JSON responses. +- Google Vertex string error body handling in RetryHelper v3.3. +- Silent result loss from HTTP actions missing timeout overrides. + +### Changed + +- DRY Aggregator SQL with extracted ResponseLogger module. +- Test suite optimization: removed ~230 redundant tests, split heavy modules, migrated 23 files from DataCase to ExUnit.Case. + ## [0.1.15] - 2026-02-21 ### Fixed diff --git a/lib/quoracle/AGENTS.md b/lib/quoracle/AGENTS.md index ad5ab6a..52e2272 100644 --- a/lib/quoracle/AGENTS.md +++ b/lib/quoracle/AGENTS.md @@ -12,7 +12,7 @@ - security/: Secret resolution and output scrubbing (added 2025-10-24) - audit/: Secret usage tracking (added 2025-10-24) - mcp/: Model Context Protocol client and configuration (added 2025-11-26) -- costs/: Agent cost tracking - AgentCost schema, Recorder, Aggregator (added 2025-12-13) +- costs/: Agent cost tracking - AgentCost schema, Recorder, Aggregator v3.0 with DRY SQL + per-model detailed queries (added 2025-12-13) ## Supervision Tree ``` diff --git a/lib/quoracle/actions/AGENTS.md b/lib/quoracle/actions/AGENTS.md index 46a8708..c810aae 100644 --- a/lib/quoracle/actions/AGENTS.md +++ b/lib/quoracle/actions/AGENTS.md @@ -18,7 +18,7 @@ - Spawn: Child agent spawning with downstream_constraints (475 lines, ConfigBuilder 273 lines, Helpers 82 lines, BudgetValidation 113 lines), dismissing flag check (v11.0), child_spawned notification (v12.0), profile parameter (v14.0), budget enforcement for budgeted parents (v17.0: :budget_required error), v19.0 removes Core.update_budget_committed callback (deadlock fix) - Wait: Unified wait parameter (142 lines, true/false/number support, interruptible) - SendMessage: Parent/child messaging (3-arity) -- DismissChild: Recursive child termination (249 lines, v4.0), async background dispatch to TreeTerminator, child_dismissed notification, budget reconciliation with absorption records and escrow release +- DismissChild: Recursive child termination (374 lines, v5.0), async background dispatch to TreeTerminator, child_dismissed notification, budget reconciliation with per-model absorption records and escrow-first release ordering - Answer (263 lines): Gemini grounding search - Model from ConfigModelSettings.get_answer_engine_model!() (config-driven) - Raises RuntimeError if answer engine model not configured @@ -133,6 +133,16 @@ Actions → Schema for parameter definitions - **Router**: Added file_read, file_write routing via ActionMapper - **CapabilityGroups**: file_read, file_write groups (allowed for all except restricted) +**Feb 23, 2026 - Cost Display & Budget Timeout Fix (WorkGroupID: fix-20260223-cost-display-budget-timeout)**: +- **DismissChild v5.0 (374 lines)**: Per-model absorption records + escrow-first ordering + - `create_absorption_records/7`: Creates one `child_budget_absorbed` record per model_spec (preserves model attribution) + - `reconcile_child_budget/8`: Escrow release BEFORE absorption records (prevents budget leak on parent crash) + - `build_absorption_metadata/4`: Includes model_spec, token counts, costs for LLM models; omits for external costs + - `query_child_tree_by_model/1`: Snapshots per-model tree costs before TreeTerminator deletes records + - `decimal_to_string/1`: Metadata formatting helper +- **ActionExecutor**: `case action_atom` timeout override — `:adjust_budget` → `:infinity` (prevents 5s Task.yield killing slow budget ops) +- **Router v32.0**: @always_sync_actions canonical 13-action list + timeout override mechanism + **Feb 17, 2026 - Adjust Budget Timeout Fix (WorkGroupID: fix-20260217-adjust-budget-timeout)**: - **AdjustBudget v3.0 (75 lines)**: Cast-based budget update, unified code path - Deleted Path A (adjust_child_directly, do_adjust, validate_adjustment) diff --git a/lib/quoracle/actions/dismiss_child.ex b/lib/quoracle/actions/dismiss_child.ex index a9ca827..a0c3143 100644 --- a/lib/quoracle/actions/dismiss_child.ex +++ b/lib/quoracle/actions/dismiss_child.ex @@ -142,17 +142,19 @@ defmodule Quoracle.Actions.DismissChild do # Snapshot budget state BEFORE termination (TreeTerminator v2.0 deletes cost records) child_budget_data = get_child_budget_data(child_id, deps.registry) - tree_spent = query_child_tree_spent(child_id) + {tree_spent, per_model} = query_child_tree_by_model(child_id) TreeTerminator.terminate_tree(child_id, parent_id, reason, deps) - # Reconcile budget: create absorption record, release escrow (v4.0) + # Reconcile budget: create per-model absorption records, release escrow (v5.0) reconcile_child_budget( + parent_id, parent_pid, child_budget_data, child_id, task_id, tree_spent, + per_model, deps.pubsub ) @@ -189,41 +191,47 @@ defmodule Quoracle.Actions.DismissChild do end end - # Query total tree spent before termination deletes records (v4.0) - @spec query_child_tree_spent(String.t()) :: Decimal.t() - defp query_child_tree_spent(child_id) do - own_costs = Aggregator.by_agent(child_id) - children_costs = Aggregator.by_agent_children(child_id) + # Query per-model tree costs before termination deletes records (v5.0) + # Returns {total_tree_spent, per_model_breakdown} for absorption record creation. + @spec query_child_tree_by_model(String.t()) :: {Decimal.t(), [map()]} + defp query_child_tree_by_model(child_id) do + per_model = Aggregator.by_agent_tree_and_model_detailed(child_id) - own_spent = own_costs.total_cost || Decimal.new("0") - children_spent = children_costs.total_cost || Decimal.new("0") - Decimal.add(own_spent, children_spent) + tree_spent = + Enum.reduce(per_model, Decimal.new("0"), fn row, acc -> + if row.total_cost, do: Decimal.add(acc, row.total_cost), else: acc + end) + + {tree_spent, per_model} end - # Reconcile budget with pre-queried spent (after tree termination) (v4.0) - # v4.1: Always create absorption record for non-zero costs to preserve task-level totals. - # Previously, N/A children had their cost records deleted by TreeTerminator with no - # replacement, causing task costs to drop. + # Reconcile budget with per-model breakdown (after tree termination) (v5.0) + # Creates per-model absorption records to preserve model attribution in Cost Details UI. + # Uses parent_id string directly (no Core.get_state call needed). @spec reconcile_child_budget( + String.t(), pid() | nil, map() | nil, String.t(), binary() | nil, Decimal.t(), + [map()], atom() | nil ) :: :ok defp reconcile_child_budget( + parent_id, parent_pid, child_budget_data, child_id, task_id, tree_spent, + per_model, pubsub ) do - # Step 1: Create absorption record for any non-zero costs (all budget modes) - create_absorption_record(parent_pid, child_budget_data, child_id, task_id, tree_spent, pubsub) - - # Step 2: Release escrow only for allocated children + # Step 1: Release escrow FIRST for allocated children (requires live parent). + # This is the volatile operation — if parent crashes between escrow release and + # absorption, the committed budget is already freed (no leak). Absorption records + # are durable DB inserts that can be retried or created even with a dead parent. case child_budget_data do %{mode: :allocated, allocated: allocated} when not is_nil(allocated) -> if parent_pid && Process.alive?(parent_pid) do @@ -238,44 +246,66 @@ defmodule Quoracle.Actions.DismissChild do :ok end + # Step 2: Create per-model absorption records (no process liveness needed) + create_absorption_records( + parent_id, + child_budget_data, + child_id, + task_id, + tree_spent, + per_model, + pubsub + ) + :ok end - # Create absorption cost record under parent to preserve task-level totals. - # TreeTerminator deletes the child's original cost records, so this record - # ensures those costs remain visible in task-level aggregation queries. - @spec create_absorption_record( - pid() | nil, + # Create per-model absorption cost records under parent to preserve task-level totals + # and model attribution. TreeTerminator deletes the child's original cost records, + # so these records ensure costs remain visible in both task-level and per-model queries. + # Uses parent_id string directly — no process liveness check or Core.get_state needed. + @spec create_absorption_records( + String.t(), map() | nil, String.t(), binary() | nil, Decimal.t(), + [map()], atom() | nil ) :: :ok - defp create_absorption_record(nil, _budget_data, _child_id, _task_id, _tree_spent, _pubsub) do + defp create_absorption_records( + _parent_id, + _budget_data, + _child_id, + _task_id, + _tree_spent, + [], + _pubsub + ) do :ok end - defp create_absorption_record(parent_pid, budget_data, child_id, task_id, tree_spent, pubsub) do - if !Decimal.equal?(tree_spent, 0) && Process.alive?(parent_pid) do - try do - {:ok, parent_state} = Core.get_state(parent_pid) - parent_agent_id = parent_state.agent_id + defp create_absorption_records( + parent_id, + budget_data, + child_id, + task_id, + tree_spent, + per_model, + pubsub + ) do + allocated = if budget_data, do: budget_data[:allocated], else: nil - allocated = if budget_data, do: budget_data[:allocated], else: nil + Enum.each(per_model, fn model_row -> + cost_usd = model_row.total_cost + if cost_usd && !Decimal.equal?(cost_usd, 0) do cost_data = %{ - agent_id: parent_agent_id, + agent_id: parent_id, task_id: task_id, cost_type: "child_budget_absorbed", - cost_usd: tree_spent, - metadata: %{ - child_agent_id: child_id, - child_allocated: format_allocated(allocated), - child_tree_spent: decimal_to_string(tree_spent), - unspent_returned: format_unspent(allocated, tree_spent), - dismissed_at: DateTime.to_iso8601(DateTime.utc_now()) - } + cost_usd: cost_usd, + metadata: build_absorption_metadata(model_row, child_id, allocated, tree_spent) } if pubsub do @@ -283,14 +313,42 @@ defmodule Quoracle.Actions.DismissChild do else Recorder.record_silent(cost_data) end - catch - :exit, _ -> :ok end - end + end) :ok end + # Build absorption metadata for a per-model record. + # Includes model_spec and token data when present (LLM costs), + # omits model_spec for external/non-model costs. + # Preserves child_tree_spent and unspent_returned for backward compatibility. + @spec build_absorption_metadata(map(), String.t(), Decimal.t() | nil, Decimal.t()) :: map() + defp build_absorption_metadata(model_row, child_id, allocated, tree_spent) do + base = %{ + "child_agent_id" => child_id, + "child_allocated" => format_allocated(allocated), + "child_tree_spent" => decimal_to_string(tree_spent), + "unspent_returned" => format_unspent(allocated, tree_spent), + "dismissed_at" => DateTime.to_iso8601(DateTime.utc_now()) + } + + if model_row.model_spec do + Map.merge(base, %{ + "model_spec" => model_row.model_spec, + "input_tokens" => to_string(model_row.input_tokens || 0), + "output_tokens" => to_string(model_row.output_tokens || 0), + "reasoning_tokens" => to_string(model_row.reasoning_tokens || 0), + "cached_tokens" => to_string(model_row.cached_tokens || 0), + "cache_creation_tokens" => to_string(model_row.cache_creation_tokens || 0), + "input_cost" => decimal_to_string(model_row.input_cost || Decimal.new("0")), + "output_cost" => decimal_to_string(model_row.output_cost || Decimal.new("0")) + }) + else + base + end + end + # Format allocated for absorption metadata @spec format_allocated(Decimal.t() | nil) :: String.t() defp format_allocated(nil), do: "N/A" diff --git a/lib/quoracle/actions/router.ex b/lib/quoracle/actions/router.ex index 02be0b4..54d1ced 100644 --- a/lib/quoracle/actions/router.ex +++ b/lib/quoracle/actions/router.ex @@ -64,8 +64,14 @@ defmodule Quoracle.Actions.Router do ) # For :call_mcp, fetch existing mcp_client from agent state (if any) before lazy-init + # Defense layer 3: liveness guard catches stale PID if Core hasn't processed DOWN yet existing_mcp_client = - if action_type == :call_mcp, do: GenServer.call(agent_pid, :get_mcp_client), else: nil + if action_type == :call_mcp do + pid = GenServer.call(agent_pid, :get_mcp_client) + if pid && Process.alive?(pid), do: pid, else: nil + else + nil + end # Build opts with agent metadata, then lazy-init MCP client only for :call_mcp base_opts = diff --git a/lib/quoracle/actions/router/AGENTS.md b/lib/quoracle/actions/router/AGENTS.md index a0d3181..679edcd 100644 --- a/lib/quoracle/actions/router/AGENTS.md +++ b/lib/quoracle/actions/router/AGENTS.md @@ -1,12 +1,12 @@ # lib/quoracle/actions/router/ ## Sub-modules (extracted for 500-line limit, Router now 496 lines) -- MCPHelpers: MCP client lazy initialization (52 lines) - Added 2025-12-16 +- MCPHelpers: MCP client lazy initialization (60 lines, v32.0: Process.alive? guard for stale PID detection) - Added 2025-12-16 - MockExecution: Mock action execution (174 lines) - WaitFlow: Wait flow control (48 lines, v26.0 simplified to no-ops - all triggers moved to Agent layer) - Metrics: Action metrics tracking (50 lines) - TaskManager: Async task lifecycle (96 lines) -- Execution: Smart mode execution (264 lines, v31.0: Logger.warning on timeout) +- Execution: Smart mode execution (265 lines, v31.0: Logger.warning on timeout, v32.0: Logger.error in catch :exit block) - ActionMapper: Action type→module mapping (29 lines) - ShellCommandManager: Shell command state management (142 lines) - Added 2025-10-14, updated 2025-10-17 - Persistence: DB persistence for action results (135 lines) - Added 2025-10-14 @@ -16,7 +16,7 @@ - Security: Secret resolution and output scrubbing (added with secret system) ## Key Functions -- MCPHelpers.get_or_init_mcp_client/1: Get existing or create new MCP client from opts +- MCPHelpers.get_or_init_mcp_client/1: Get existing or create new MCP client from opts (v32.0: Process.alive? guard re-initializes dead clients) - MCPHelpers.maybe_lazy_init_mcp_client/2: Only initializes for :call_mcp action type - MockExecution.execute_action_mock/5: Simulates action responses - WaitFlow.handle_immediate/3: Timer notification only for timed waits (no triggers - v26.0) diff --git a/lib/quoracle/actions/router/execution.ex b/lib/quoracle/actions/router/execution.ex index a8e0c79..e4dcef6 100644 --- a/lib/quoracle/actions/router/execution.ex +++ b/lib/quoracle/actions/router/execution.ex @@ -104,6 +104,7 @@ defmodule Quoracle.Actions.Router.Execution do {:error, {:action_crashed, Exception.message(e)}} catch :exit, reason -> + Logger.error("Action execution crashed (exit): #{inspect(reason)}") {:error, {:action_crashed, reason}} end end) diff --git a/lib/quoracle/actions/router/mcp_helpers.ex b/lib/quoracle/actions/router/mcp_helpers.ex index d9103c1..8b3b26f 100644 --- a/lib/quoracle/actions/router/mcp_helpers.ex +++ b/lib/quoracle/actions/router/mcp_helpers.ex @@ -17,11 +17,13 @@ defmodule Quoracle.Actions.Router.MCPHelpers do require Logger + needs_init = (is_nil(mcp_client) or not Process.alive?(mcp_client)) and is_pid(agent_pid) + Logger.debug( - "MCPHelpers: mcp_client=#{inspect(mcp_client)}, agent_pid=#{inspect(agent_pid)}, creating_new=#{is_nil(mcp_client) and is_pid(agent_pid)}" + "MCPHelpers: mcp_client=#{inspect(mcp_client)}, agent_pid=#{inspect(agent_pid)}, creating_new=#{needs_init}" ) - if is_nil(mcp_client) and is_pid(agent_pid) do + if needs_init do agent_id = Keyword.get(opts, :agent_id) || GenServer.call(agent_pid, :get_agent_id) mcp_opts = diff --git a/lib/quoracle/actions/send_message.ex b/lib/quoracle/actions/send_message.ex index 591f866..9ae0f8a 100644 --- a/lib/quoracle/actions/send_message.ex +++ b/lib/quoracle/actions/send_message.ex @@ -181,13 +181,12 @@ defmodule Quoracle.Actions.SendMessage do # Look up agent's composite value to get parent info case Registry.lookup(registry, {:agent, agent_id}) do [{_pid, composite}] when is_map(composite) -> - parent_pid = Map.get(composite, :parent_pid) parent_id = Map.get(composite, :parent_id) # Has parent - look up parent to verify it exists if parent_id do case Registry.lookup(registry, {:agent, parent_id}) do - [{^parent_pid, _}] -> {:ok, [{:agent, parent_pid, parent_id}]} + [{pid, _}] -> {:ok, [{:agent, pid, parent_id}]} _ -> {:error, :parent_not_found} end else diff --git a/lib/quoracle/actions/spawn.ex b/lib/quoracle/actions/spawn.ex index a15c941..5fbf656 100644 --- a/lib/quoracle/actions/spawn.ex +++ b/lib/quoracle/actions/spawn.ex @@ -152,7 +152,7 @@ defmodule Quoracle.Actions.Spawn do {:error, "Budget is required when spawning children. " <> "Specify a budget amount (e.g., budget: \"50.00\"). " <> - "Use get_budget to check your available funds."} + "Check your context for available funds."} error -> error diff --git a/lib/quoracle/actions/spawn/budget_validation.ex b/lib/quoracle/actions/spawn/budget_validation.ex index 7f6aade..7994797 100644 --- a/lib/quoracle/actions/spawn/budget_validation.ex +++ b/lib/quoracle/actions/spawn/budget_validation.ex @@ -33,7 +33,7 @@ defmodule Quoracle.Actions.Spawn.BudgetValidation do # N/A, nil, or unknown parent - child gets N/A budget (unlimited) {:ok, %{ - child_budget_data: %{mode: :na, allocated: nil, committed: nil}, + child_budget_data: Quoracle.Budget.Schema.new_na(), escrow_amount: nil }} end diff --git a/lib/quoracle/agent/AGENTS.md b/lib/quoracle/agent/AGENTS.md index d9414cd..b3f2503 100644 --- a/lib/quoracle/agent/AGENTS.md +++ b/lib/quoracle/agent/AGENTS.md @@ -1,7 +1,7 @@ # lib/quoracle/agent/ ## Modules -- Core: Event-driven GenServer (490 lines), delegates message handling, stores prompt_fields + dismissing flag + capability_groups + shell_routers in state, v20.0 extracts adjust_child_budget/update_budget_data to ClientAPI, adds route_to_shell_router/3 helper, v35.0 adds spawn_failed delegation to MessageInfoHandler, v37.0 adds handle_cast({:set_budget_allocated, ...}) delegation to BudgetHandler +- Core: Event-driven GenServer (497 lines), delegates message handling, stores prompt_fields + dismissing flag + capability_groups + shell_routers + cached_system_prompt in state, v20.0 extracts adjust_child_budget/update_budget_data to ClientAPI, adds route_to_shell_router/3 helper, v35.0 adds spawn_failed delegation to MessageInfoHandler, v37.0 adds handle_cast({:set_budget_allocated, ...}) delegation to BudgetHandler, v38.0 adds invalidate_cached_system_prompt/1 private helper (learn_skills invalidation) - Core.ClientAPI: GenServer wrappers (209 lines, 17 functions with @spec), v20.0 adds adjust_child_budget/4, update_budget_data/2, v34.0 adds release_child_budget/3 - Core.TodoHandler: Todo list state management (57 lines), extracted for 500-line limit - Core.BudgetHandler: Budget GenServer callbacks (247 lines), adjust_child_budget/4 (v37.0: cast-based, no child calls, spent-only decrease validation), handle_set_budget_allocated/2 (v37.0), handle_release_child_budget/3 (v34.0), update_over_budget_status/1 (non-monotonic since v34.0) @@ -9,24 +9,25 @@ - Core.Initialization: Init and DB setup (154 lines), extracted for 500-line limit (2025-10-17) - Core.Persistence: DB persistence (149 lines), extract_parent_agent_id with Registry→state.parent_id fallback (v36.0), delegates ACE to submodule - Core.Persistence.ACEState: ACE state serialization (332 lines), context_lessons + model_states + model_histories (v5.0), extracted for 500-line limit -- Core.MessageInfoHandler: Info message dispatch (329 lines), handle_wait_expired/2 with v21.0 staleness check, handle_trigger_consensus/1 (v19.0 unified handler), handle_agent_message_2tuple/3tuple, handle_down/4, handle_exit/3, handle_spawn_failed/2 (v35.0: logs warning, records failure in history, removes child, schedules consensus) +- Core.MessageInfoHandler: Info message dispatch (333 lines), handle_wait_expired/2 with v21.0 staleness check, handle_trigger_consensus/1 (v19.0 unified handler), handle_agent_message_2tuple/3tuple, handle_down/4 (v38.0: MCP client DOWN clears state.mcp_client), handle_exit/3, handle_spawn_failed/2 (v35.0: logs warning, records failure in history, removes child, schedules consensus) - RegistryQueries: Registry queries (77 lines), composite value extraction - MessageHandler: Message processing (377 lines, was 591 before ActionResultHandler extraction), timer cancellation (R11-R13), consensus integration, NO_EXECUTE action_type tracking, delegates to ConsensusHandler (v9.0), routes images via ImageDetector (v11.0), message queueing (v12.0), v13.0 handles 3-tuple via StateUtils.merge_consensus_state, v15.0 unified run_consensus_cycle/2, handle_consensus_error/4 DRY helper, v16.0 deferred consensus via consensus_scheduled flag, v18.0 deferred consensus for idle agents, v24.0 delegates handle_action_result/4 to ActionResultHandler -- MessageHandler.ActionResultHandler: Action result processing (382 lines, extracted REFACTOR 2026-02-13), handle_action_result/4 with extended wait parameter handling, handle_batch_action_result/4, flush_queued_messages/1, format_sender_id/1, maybe_track_child/3 (spawn_child tracking), maybe_update_budget_committed/3 (replaces Core.update_budget_committed callback), v25.0: maybe_track_shell_router/3 (shell_routers population), error-aware continuation guard, v26.0: async_shell_phase1?/1 shared predicate (pending_actions guard + shell_routers tracking), v27.0: maybe_schedule_consensus/1 DRY helper for consensus deferral when self_contained actions pending +- MessageHandler.ActionResultHandler: Action result processing (383 lines, extracted REFACTOR 2026-02-13), handle_action_result/4 with extended wait parameter handling, handle_batch_action_result/4, flush_queued_messages/1, format_sender_id/1, maybe_track_child/3 (spawn_child tracking), maybe_update_budget_committed/3 (replaces Core.update_budget_committed callback), v25.0: maybe_track_shell_router/3 (shell_routers population), error-aware continuation guard, v26.0: async_shell_phase1?/1 shared predicate (pending_actions guard + shell_routers tracking), v27.0: maybe_schedule_consensus/1 DRY helper for consensus deferral when self_contained actions pending, v28.0: LogHelper.log_action_error wiring for server-side error logging - ImageDetector: Image detection from action results (167 lines), converts MCP screenshots to multimodal content, supports base64 and URL images -- Consensus: Multi-LLM consensus (494 lines), pre-clustering validation filter (v7.0), per-model refinement context (v10.0), system prompt injection fix, v19.0 threads max_refinement_rounds from state to context +- Consensus: Multi-LLM consensus (495 lines), pre-clustering validation filter (v7.0), per-model refinement context (v10.0), system prompt injection fix, v19.0 threads max_refinement_rounds from state to context. v18.0 cache via opts passthrough +- Consensus.ResponseLogger: Response logging helpers (40 lines), slim_responses_for_logging/1 (extracted from Consensus in REFACTOR fix-20260223-cost-display-budget-timeout for 500-line limit) - TokenManager: Token counting (376 lines), tiktoken integration via Tiktoken.CL100K for accurate BPE tokenization (v5.0), v8.0 adds history_tokens_for_model/2 helper, v16.0 adds estimate_all_messages_tokens/1 (all messages including system) and get_model_output_limit/1 (LLMDB limits.output) - ContextManager: History summarization (274 lines), builds field-based prompts for consensus, JSON formatting for :decision/:result entries (v2.0), 1-arity build_conversation_messages DELETED (v5.0), v7.0 ACE injection removed (now in AceInjector) - ConfigManager: Config normalization (500 lines), atomic registration, ModelPoolInit submodule extracted, v5.0 preserves model_histories from restoration config, v8.0 extracts capability_groups, v11.0 extracts max_refinement_rounds - ConfigManager.ModelPoolInit: Model pool initialization (37 lines), get_model_pool_for_init/2, initialize_model_histories/1 -- ConsensusHandler: Consensus execution (244 lines), v20.0 single prompt_opts for UI/LLM consistency (fix-20260113-skill-injection), extracts active_skills + skills_path from state +- ConsensusHandler: Consensus execution (269 lines), v20.0 single prompt_opts for UI/LLM consistency (fix-20260113-skill-injection), extracts active_skills + skills_path from state, v27.0 lazy-build system prompt cache with fast-path guard - ConsensusHandler.Helpers: Helper functions (116 lines), normalize_sibling_context/1, self_contained_actions/0, has_pending_self_contained?/1 (v26.0), coerce_wait_value/1 (v34.0 DRY extraction), prepend_to_content/2, extract_shell_check_id/2 (v25.0 shared helper for shell routing) -- ConsensusHandler.LogHelper: Logging helpers (40 lines), safe_broadcast_log/5, log_action_error/1 (extracted for 500-line limit) +- ConsensusHandler.LogHelper: Logging helpers (63 lines), safe_broadcast_log/5, log_action_error/1 (v28.0: {:error, _} unwrap, {:action_crashed, tuple} clause, extended @warning_errors with transient network errors) - ConsensusHandler.TodoInjector: TODO injection (82 lines), inject_todo_context/2, format_todos_as_xml/1, escape_xml/1 - ConsensusHandler.ChildrenInjector: Children context injection (78 lines), inject_children_context/2, format_children/1, Registry-based status check - ConsensusHandler.AceInjector: ACE context injection (82 lines, v1.0), inject_ace_context/3 into FIRST user message (historical knowledge), format_ace_context/2 - ConsensusHandler.ContextInjector: Context token injection (95 lines, v2.0), inject_context_tokens/1 counts fully-built messages (excluding system prompt), format_context_tokens/1 with comma separators -- StateUtils: State manipulation helpers (145 lines), action_type tracking for NO_EXECUTE, v3.0 adds merge_consensus_state/2 for ACE state merging (extracted from MessageHandler/ConsensusContinuationHandler), v5.0 adds cancel_wait_timer/1 for DRY timer cancellation (4 pattern-matched clauses), v6.0 adds schedule_consensus_continuation/1 for DRY "set flag + send trigger" pattern +- StateUtils: State manipulation helpers (145 lines), action_type tracking for NO_EXECUTE, v3.0 adds merge_consensus_state/2 for ACE state merging (extracted from MessageHandler/ConsensusContinuationHandler), v5.0 adds cancel_wait_timer/1 for DRY timer cancellation (4 pattern-matched clauses), v6.0 adds schedule_consensus_continuation/1 for DRY "set flag + send trigger" pattern, v38.0 adds :cached_system_prompt to merge_keys - ConsensusContinuationHandler: Continuation handling (59 lines), v4.0 delegates to MessageHandler.run_consensus_cycle/2 for unified message flush, v5.0 delegates cancel_wait_timer to StateUtils - DynSup: DynamicSupervisor wrapper (185 lines), 2-arity terminate for tests, v3.0 restore_agent prefers ace_state.model_histories, v7.0 restores max_refinement_rounds from profile - Reflector: LLM-based lesson/state extraction (321 lines), self-reflection pattern, JSON parsing with validation, v2.1 uses ContentStringifier for multimodal prompts, v3.0 retries malformed LLM responses (empty/non-JSON/invalid schema) with shared budget via retry_ctx map diff --git a/lib/quoracle/agent/config_manager.ex b/lib/quoracle/agent/config_manager.ex index fcd1c7c..a6593cf 100644 --- a/lib/quoracle/agent/config_manager.ex +++ b/lib/quoracle/agent/config_manager.ex @@ -43,6 +43,14 @@ defmodule Quoracle.Agent.ConfigManager do test_opts end + # Add force_persist to test_opts if it's at the top level + test_opts = + if Map.has_key?(config, :force_persist) do + Keyword.put(test_opts, :force_persist, config.force_persist) + else + test_opts + end + # Extract sandbox_owner from config or test_opts sandbox_owner = Map.get(config, :sandbox_owner) || diff --git a/lib/quoracle/agent/consensus.ex b/lib/quoracle/agent/consensus.ex index 9f665ce..5b39c2e 100644 --- a/lib/quoracle/agent/consensus.ex +++ b/lib/quoracle/agent/consensus.ex @@ -13,6 +13,7 @@ defmodule Quoracle.Agent.Consensus do alias Quoracle.Agent.Consensus.{ MockResponseGenerator, + ResponseLogger, TestMode, PerModelQuery, SystemPromptInjector @@ -61,6 +62,22 @@ defmodule Quoracle.Agent.Consensus do @spec get_consensus_with_state(map(), keyword()) :: {:ok, consensus_result(), map()} | {:error, atom()} def get_consensus_with_state(state, opts) do + # Fast path: In test mode without simulate flags, return mock result directly + # Skips the entire query → parse → validate → cluster pipeline + # Still validates model_histories exists (preserves error contract) + cond do + Map.get(state, :model_histories) == nil -> + {:error, :missing_model_histories} + + TestMode.fast_test_path?(state, opts) -> + {:ok, {:consensus, TestMode.mock_consensus_action(), [per_model_queries: true]}, state} + + true -> + get_consensus_with_state_impl(state, opts) + end + end + + defp get_consensus_with_state_impl(state, opts) do # Validate model_histories field exists case Map.get(state, :model_histories) do nil -> @@ -73,6 +90,11 @@ defmodule Quoracle.Agent.Consensus do max_rounds = Map.get(state, :max_refinement_rounds, 4) opts = Keyword.put_new(opts, :max_refinement_rounds, max_rounds) + # Build system prompt ONCE and cache in opts for all downstream calls. + # Without this, PromptBuilder.build_system_prompt_with_context is called + # per model per round (N*R times), each doing 2 DB queries + schema generation. + opts = ensure_cached_system_prompt(state, opts) + # Query each model with its own history case query_models_with_per_model_histories(state, model_pool, opts) do {:ok, responses, updated_state} -> @@ -82,7 +104,7 @@ defmodule Quoracle.Agent.Consensus do opts[:agent_id], :debug, "Received #{length(responses)} LLM responses (initial round)", - %{raw_responses: slim_responses_for_logging(responses)}, + %{raw_responses: ResponseLogger.slim_responses_for_logging(responses)}, opts[:pubsub] ) end @@ -310,7 +332,7 @@ defmodule Quoracle.Agent.Consensus do opts[:agent_id], :debug, "Received #{length(refined_responses)} LLM responses (refinement round #{round + 1})", - %{raw_responses: slim_responses_for_logging(refined_responses)}, + %{raw_responses: ResponseLogger.slim_responses_for_logging(refined_responses)}, opts[:pubsub] ) end @@ -354,6 +376,28 @@ defmodule Quoracle.Agent.Consensus do defp user_entry?(%{role: "user"}), do: true defp user_entry?(_), do: false + # Build system prompt once for the entire consensus process. + # If already in opts (from ConsensusHandler), keep it. + # If in state (from Core's lazy cache), use it. + # Otherwise build fresh and cache in opts. + defp ensure_cached_system_prompt(state, opts) do + if Keyword.has_key?(opts, :cached_system_prompt) do + opts + else + prompt = + Map.get(state, :cached_system_prompt) || + build_system_prompt_for_consensus(state, opts) + + Keyword.put(opts, :cached_system_prompt, prompt) + end + end + + defp build_system_prompt_for_consensus(state, opts) do + field_prompts = %{system_prompt: Map.get(state, :system_prompt)} + prompt_opts = Keyword.put(opts, :field_prompts, field_prompts) + Quoracle.Consensus.PromptBuilder.build_system_prompt_with_context(prompt_opts) + end + # v24.0: Added :cost_accumulator for embedding cost batching (feat-20260203-194408) defp extract_cost_opts(context) do opts = Map.get(context, :original_opts, []) @@ -395,7 +439,7 @@ defmodule Quoracle.Agent.Consensus do :debug, "Received #{length(result.successful_responses)} LLM responses", %{ - raw_responses: slim_responses_for_logging(result.successful_responses), + raw_responses: ResponseLogger.slim_responses_for_logging(result.successful_responses), failed_models: result.failed_models, total_latency_ms: result.total_latency_ms, aggregate_usage: result.aggregate_usage @@ -420,37 +464,6 @@ defmodule Quoracle.Agent.Consensus do defp maybe_put(map, _key, nil), do: map defp maybe_put(map, key, value), do: Map.put(map, key, value) - # Extracts only the fields needed for UI display from ReqLLM.Response objects. - # Drops the `context` field which contains the full conversation history and - # causes O(n²) memory growth when stored in log metadata. - defp slim_responses_for_logging(responses) when is_list(responses) do - Enum.map(responses, &slim_single_response/1) - end - - defp slim_single_response(%ReqLLM.Response{} = response) do - # Keep all UI-needed fields, drop only the massive context field - %{ - model: response.model, - usage: response.usage, - text: ReqLLM.Response.text(response), - finish_reason: response.finish_reason, - latency_ms: Map.get(response, :latency_ms) - } - end - - defp slim_single_response(response) when is_map(response) do - # Fallback for non-struct responses (tests, legacy) - # Preserve all fields except context to maintain UI compatibility - response - |> Map.drop([:context, "context"]) - |> Map.put_new(:text, response[:content] || response["content"]) - end - - defp slim_single_response(other) do - # Catch-all for nil, error tuples, or unexpected types - %{model: nil, usage: nil, text: inspect(other), finish_reason: nil, latency_ms: nil} - end - @doc "Parse JSON response from LLM into action map. Delegates to ActionParser." @spec parse_json_response(String.t()) :: {:ok, action_response()} | {:error, atom()} defdelegate parse_json_response(json_string), to: ActionParser diff --git a/lib/quoracle/agent/consensus/AGENTS.md b/lib/quoracle/agent/consensus/AGENTS.md index 5cc5c7e..5b48760 100644 --- a/lib/quoracle/agent/consensus/AGENTS.md +++ b/lib/quoracle/agent/consensus/AGENTS.md @@ -6,7 +6,7 @@ - PerModelQuery: Per-model query functions for consensus, 371 lines (v15.0: Condensation extraction, v16.0: dynamic max_tokens calculation) - PerModelQuery.Helpers: Extracted helper functions, 106 lines (v14.0: uses ContentStringifier for multimodal content, context length error detection) - PerModelQuery.Condensation: ACE condensation logic, 186 lines (v15.0: extracted from PerModelQuery for <500 line limit) -- SystemPromptInjector: System prompt injection for messages, 142 lines (v13.0: human prompt JSON with from:"parent") +- SystemPromptInjector: System prompt injection for messages, 80 lines (v13.0: human prompt JSON with from:"parent", v38.0: cached_system_prompt check in opts before PromptBuilder call) ## TestMode Functions - enabled?/1: keyword→boolean, checks test_mode or test flags diff --git a/lib/quoracle/agent/consensus/message_builder.ex b/lib/quoracle/agent/consensus/message_builder.ex index e6169bd..7394bd5 100644 --- a/lib/quoracle/agent/consensus/message_builder.ex +++ b/lib/quoracle/agent/consensus/message_builder.ex @@ -78,7 +78,12 @@ defmodule Quoracle.Agent.Consensus.MessageBuilder do # Step 8: Inject context token count LAST (at end of last user message) # Counts all non-system message tokens from steps 1-7 - ContextInjector.inject_context_tokens(messages) + # Skip in lightweight query mode (eliminates tiktoken BPE encoding) + if Keyword.get(opts, :skip_context_tokens, false) do + messages + else + ContextInjector.inject_context_tokens(messages) + end end @doc """ diff --git a/lib/quoracle/agent/consensus/per_model_query.ex b/lib/quoracle/agent/consensus/per_model_query.ex index 93e4d35..d0cab89 100644 --- a/lib/quoracle/agent/consensus/per_model_query.ex +++ b/lib/quoracle/agent/consensus/per_model_query.ex @@ -54,15 +54,27 @@ defmodule Quoracle.Agent.Consensus.PerModelQuery do end defp query_model_with_retry_logic(state, model_id, opts) do + lightweight? = lightweight_test_query?(opts) + + # Skip token counting in message builder for lightweight mode + build_opts = + if lightweight?, do: Keyword.put(opts, :skip_context_tokens, true), else: opts + # Use unified message-building helper (R65-R70) - messages_with_system = build_query_messages(state, model_id, opts) + messages_with_system = build_query_messages(state, model_id, build_opts) + + # In lightweight mode, skip proactive condensation + dynamic max_tokens + # (eliminates tiktoken BPE encoding + LLMDB scans per model per round) + {messages_with_system, state, dynamic_max_tokens} = + if lightweight? do + {messages_with_system, state, @output_floor} + else + {condensed_msgs, condensed_state} = + maybe_proactive_condense(messages_with_system, state, model_id, opts) - # Dynamic max_tokens: proactive condensation if available output below floor - {messages_with_system, state} = - maybe_proactive_condense(messages_with_system, state, model_id, opts) + {condensed_msgs, condensed_state, calculate_max_tokens(condensed_msgs, model_id)} + end - # Calculate dynamic max_tokens from built messages - dynamic_max_tokens = calculate_max_tokens(messages_with_system, model_id) query_opts = build_query_options(model_id, Keyword.put(opts, :max_tokens, dynamic_max_tokens)) # Support injectable model_query_fn for testing @@ -226,10 +238,15 @@ defmodule Quoracle.Agent.Consensus.PerModelQuery do end else # Production path - query each model, threading state through + lightweight? = lightweight_test_query?(opts) + {results, final_state} = Enum.map_reduce(model_pool, state, fn model_id, acc_state -> - # Check condensation first (updates state if needed) - pre_query_state = maybe_condense_for_model(acc_state, model_id, opts) + # Skip condensation check for lightweight test queries (no tiktoken/LLMDB) + pre_query_state = + if lightweight?, + do: acc_state, + else: maybe_condense_for_model(acc_state, model_id, opts) # Query returns state to propagate any condensation from retry path {result, post_query_state} = @@ -368,6 +385,15 @@ defmodule Quoracle.Agent.Consensus.PerModelQuery do defp test_mode?(opts), do: Keyword.get(opts, :test_mode, false) + # Lightweight query path: test mode with injected query fn. + # Skips tiktoken BPE encoding, LLMDB scans, and condensation checks + # since test query functions don't depend on accurate token management. + # Tests that specifically verify token management pass force_token_management: true. + defp lightweight_test_query?(opts) do + test_mode?(opts) && Keyword.has_key?(opts, :model_query_fn) && + !Keyword.get(opts, :force_token_management, false) + end + defp build_test_options(opts) do %{ test_mode: true, diff --git a/lib/quoracle/agent/consensus/response_logger.ex b/lib/quoracle/agent/consensus/response_logger.ex new file mode 100644 index 0000000..4a02ca8 --- /dev/null +++ b/lib/quoracle/agent/consensus/response_logger.ex @@ -0,0 +1,40 @@ +defmodule Quoracle.Agent.Consensus.ResponseLogger do + @moduledoc """ + Helpers for logging LLM responses during consensus. + Extracts only UI-needed fields from ReqLLM.Response objects to prevent + O(n^2) memory growth from storing full conversation context in log metadata. + """ + + @doc """ + Slim a list of LLM responses for logging, dropping the massive context field. + """ + @spec slim_responses_for_logging([any()]) :: [map()] + def slim_responses_for_logging(responses) when is_list(responses) do + Enum.map(responses, &slim_single_response/1) + end + + @spec slim_single_response(any()) :: map() + defp slim_single_response(%ReqLLM.Response{} = response) do + # Keep all UI-needed fields, drop only the massive context field + %{ + model: response.model, + usage: response.usage, + text: ReqLLM.Response.text(response), + finish_reason: response.finish_reason, + latency_ms: Map.get(response, :latency_ms) + } + end + + defp slim_single_response(response) when is_map(response) do + # Fallback for non-struct responses (tests, legacy) + # Preserve all fields except context to maintain UI compatibility + response + |> Map.drop([:context, "context"]) + |> Map.put_new(:text, response[:content] || response["content"]) + end + + defp slim_single_response(other) do + # Catch-all for nil, error tuples, or unexpected types + %{model: nil, usage: nil, text: inspect(other), finish_reason: nil, latency_ms: nil} + end +end diff --git a/lib/quoracle/agent/consensus/system_prompt_injector.ex b/lib/quoracle/agent/consensus/system_prompt_injector.ex index f59b99b..8828a17 100644 --- a/lib/quoracle/agent/consensus/system_prompt_injector.ex +++ b/lib/quoracle/agent/consensus/system_prompt_injector.ex @@ -16,12 +16,16 @@ defmodule Quoracle.Agent.Consensus.SystemPromptInjector do @doc "Ensures messages have combined system prompt with field-based configuration." @spec ensure_system_prompts(list(map()), map(), keyword()) :: list(map()) def ensure_system_prompts(messages, field_prompts, opts) do - # Pass field_prompts through opts to build_system_prompt_with_context - opts_with_fields = Keyword.put(opts, :field_prompts, field_prompts) - - # Build single integrated system prompt + # v38.0: Use cached system prompt if available, otherwise build fresh integrated_system_prompt = - PromptBuilder.build_system_prompt_with_context(opts_with_fields) + case Keyword.get(opts, :cached_system_prompt) do + nil -> + opts_with_fields = Keyword.put(opts, :field_prompts, field_prompts) + PromptBuilder.build_system_prompt_with_context(opts_with_fields) + + cached -> + cached + end # Separate action schema prompts (to be replaced) from additional context (to be preserved) {_action_schema_msgs, other_system_msgs} = diff --git a/lib/quoracle/agent/consensus/test_mode.ex b/lib/quoracle/agent/consensus/test_mode.ex index 69aa444..76cf742 100644 --- a/lib/quoracle/agent/consensus/test_mode.ex +++ b/lib/quoracle/agent/consensus/test_mode.ex @@ -76,9 +76,63 @@ defmodule Quoracle.Agent.Consensus.TestMode do end end + @doc """ + Checks if the consensus fast-path should be used. + + Returns true when test_mode is active and no flags require the real pipeline + (simulate_*, model_query_fn, seed_action, track_temperatures, force_condense). + """ + @spec fast_test_path?(map(), keyword()) :: boolean() + def fast_test_path?(state, opts) do + Map.get(state, :test_mode, false) && + !Keyword.has_key?(opts, :model_query_fn) && + !Keyword.get(opts, :simulate_failure, false) && + !Keyword.has_key?(opts, :seed_action) && + !Keyword.get(opts, :track_temperatures, false) && + !Keyword.get(opts, :force_condense, false) && + !Keyword.get(opts, :force_persist, false) && + !any_simulate_flags?(opts) + end + + @doc """ + Returns the pre-built mock consensus action for the fast test path. + Matches the orient action that MockResponseGenerator returns by default. + """ + @spec mock_consensus_action() :: map() + def mock_consensus_action do + %{ + action: :orient, + params: %{ + "current_situation" => "Processing task", + "goal_clarity" => "Clear objectives", + "available_resources" => "Full capabilities", + "key_challenges" => "None identified", + "delegation_consideration" => "none" + }, + reasoning: "Mock consensus (fast path)", + confidence: 1.0, + wait: true, + _fast_path: true + } + end + # Private helpers defp has_test_flags?(opts) do Enum.any?(@test_flags, &Keyword.has_key?(opts, &1)) end + + defp any_simulate_flags?(opts) do + Enum.any?( + [ + :simulate_tie, + :simulate_no_majority, + :force_no_consensus, + :force_max_rounds, + :simulate_refinement_failure, + :simulate_partial_failure + ], + &Keyword.get(opts, &1, false) + ) + end end diff --git a/lib/quoracle/agent/consensus_handler.ex b/lib/quoracle/agent/consensus_handler.ex index b21d652..358d00b 100644 --- a/lib/quoracle/agent/consensus_handler.ex +++ b/lib/quoracle/agent/consensus_handler.ex @@ -114,20 +114,45 @@ defmodule Quoracle.Agent.ConsensusHandler do skills_path: Map.get(state, :skills_path) ] - # Calculate per-model message counts by actually building messages - # This accounts for merging consecutive same-role messages - per_model_counts = - Enum.map(models_list, fn model_id -> - # Build messages as PerModelQuery will (includes merging) - messages = Quoracle.Agent.ContextManager.build_conversation_messages(state, model_id) + # v38.0: Lazy-build and cache system prompt for reuse across consensus cycles. + # Skip caching on fast-path (Option E) since it bypasses the entire pipeline. + {prompt_opts, state} = + if Quoracle.Agent.Consensus.TestMode.fast_test_path?(state, consensus_opts) do + {prompt_opts, state} + else + case Map.get(state, :cached_system_prompt) do + nil -> + # Include field_prompts so the agent's role/cognitive_style/constraints + # are embedded in the identity section of the cached prompt + cache_build_opts = + prompt_opts ++ + [ + field_prompts: %{system_prompt: Map.get(state, :system_prompt)}, + sandbox_owner: Map.get(state, :sandbox_owner) + ] + + prompt = + Quoracle.Consensus.PromptBuilder.build_system_prompt_with_context(cache_build_opts) + + {Keyword.put(prompt_opts, :cached_system_prompt, prompt), + Map.put(state, :cached_system_prompt, prompt)} + + existing -> + {Keyword.put(prompt_opts, :cached_system_prompt, existing), state} + end + end - # Add system prompt (+1) - user_prompt no longer injected (flows through history) - length(messages) + 1 - end) + if is_atom(pubsub) and pubsub != :test_pubsub do + # Calculate per-model message counts by actually building messages + # This accounts for merging consecutive same-role messages + per_model_counts = + Enum.map(models_list, fn model_id -> + messages = Quoracle.Agent.ContextManager.build_conversation_messages(state, model_id) + length(messages) + 1 + end) - counts_str = Enum.join(per_model_counts, "/") + counts_str = Enum.join(per_model_counts, "/") - if is_atom(pubsub) and pubsub != :test_pubsub do # Build sent_messages for UI using shared MessageBuilder sent_messages = MessageBuilder.build_messages_for_models(state, models_list, prompt_opts) diff --git a/lib/quoracle/agent/consensus_handler/AGENTS.md b/lib/quoracle/agent/consensus_handler/AGENTS.md index 5cb8bd3..2e90a66 100644 --- a/lib/quoracle/agent/consensus_handler/AGENTS.md +++ b/lib/quoracle/agent/consensus_handler/AGENTS.md @@ -1,9 +1,9 @@ # lib/quoracle/agent/consensus_handler/ ## Modules -- ActionExecutor: Non-blocking consensus action execution (346 lines), dispatches Router.execute via Task.Supervisor, result returns via GenServer.cast, v36.0 outer try/rescue/catch crash protection + MCP sync timeout +- ActionExecutor: Non-blocking consensus action execution (363 lines), dispatches Router.execute via Task.Supervisor, result returns via GenServer.cast, v36.0 outer try/rescue/catch crash protection + MCP sync timeout, timeout overrides: :adjust_budget → :infinity, :call_mcp → 600_000, :answer_engine → 120_000, :fetch_web → 60_000, :call_api → 120_000, :generate_images → 300_000 (v39.0) - Helpers: Shared helper functions (116 lines), self_contained_actions/0, has_pending_self_contained?/1 (v26.0), coerce_wait_value/1, extract_shell_check_id/2, normalize_sibling_context/1 -- LogHelper: Logging helpers (40 lines), safe_broadcast_log/5, log_action_error/1 +- LogHelper: Logging helpers (63 lines), safe_broadcast_log/5, log_action_error/1 (v28.0: {:error, _} unwrap, {:action_crashed, tuple} clause, extended @warning_errors) - TodoInjector: Todo list context injection (82 lines), inject_todo_context/2 - ChildrenInjector: Children context injection (78 lines), inject_children_context/2 - AceInjector: ACE context injection (82 lines), inject_ace_context/3 @@ -13,7 +13,7 @@ - ActionExecutor.execute_consensus_action/3: Entry point, validates wait params, dispatches action - ActionExecutor.dispatch_action/9: Task.Supervisor.start_child with outer try/rescue/catch, Router.execute in background, casts result to Core, crash_in_task injection for testing (v36.0) - ActionExecutor.spawn_and_monitor_router/4: Spawn Router + Process.monitor + active_routers tracking (v25.0) -- ActionExecutor MCP sync timeout: Forces 600_000ms timeout for :call_mcp actions (v36.0, prevents smart_threshold async dispatch) +- ActionExecutor timeout overrides: Forces 600_000ms for :call_mcp (v36.0), :infinity for :adjust_budget (fix-20260223), 120_000ms for :answer_engine, 60_000ms for :fetch_web, 120_000ms for :call_api, 300_000ms for :generate_images (v39.0 — prevents 100ms smart_threshold from losing HTTP action results) - Helpers.extract_shell_check_id/2: Detect check_id in shell params for routing through existing Router (v25.0) - Helpers.self_contained_actions/0: 10 actions that complete instantly (wait:true would stall) - Helpers.has_pending_self_contained?/1: Check if any pending_actions are self_contained (v26.0, used by ActionResultHandler.maybe_schedule_consensus/1) @@ -26,7 +26,7 @@ 4. Route check_id via Helpers.extract_shell_check_id → lookup shell_routers → use existing Router 5. For normal actions: spawn_and_monitor_router (spawn + monitor + active_routers) 6. Add to pending_actions -7. For :call_mcp actions: inject 600_000ms timeout to force sync execution (v36.0) +7. Timeout overrides: :call_mcp → 600_000ms (v36.0), :adjust_budget → :infinity (fix-20260223) 8. dispatch_action → Task.Supervisor.start_child → try/rescue/catch → Router.execute → cast result to Core 9. Return state immediately (Core is free for GenServer.call) diff --git a/lib/quoracle/agent/consensus_handler/action_executor.ex b/lib/quoracle/agent/consensus_handler/action_executor.ex index 095c703..1efe57b 100644 --- a/lib/quoracle/agent/consensus_handler/action_executor.ex +++ b/lib/quoracle/agent/consensus_handler/action_executor.ex @@ -99,8 +99,10 @@ defmodule Quoracle.Agent.ConsensusHandler.ActionExecutor do # Add decision to history using StateUtils (with corrected wait value) state = StateUtils.add_history_entry(state, :decision, action_response) - # Persist updated conversation history to database - Core.persist_conversation(state) + # Persist updated conversation history to database (skip in test mode for speed) + unless Map.get(state, :test_mode, false) && !force_persist?(state) do + Core.persist_conversation(state) + end # Generate action ID using current counter # Use Map.get for optional fields (works with both structs and maps) @@ -112,14 +114,26 @@ defmodule Quoracle.Agent.ConsensusHandler.ActionExecutor do execute_opts = build_execute_opts(state, action_id, agent_pid, action_response) - # v36.0: Force synchronous execution for MCP actions. - # MCP retry delays (500ms+) exceed the 100ms smart_threshold, causing async dispatch - # which loses results. A 10-minute timeout covers worst-case retry scenarios. + # Force synchronous execution for actions whose latency exceeds the 100ms + # smart_threshold. Without an explicit timeout the Router enters "smart mode", + # yields for only 100ms, and returns {:async_task, ...}. The ActionExecutor + # background Task then casts that opaque tuple as the "result", the agent + # removes the action from pending_actions, and the *real* result arriving + # later is silently discarded (unknown action_id). + # + # v36.0: :call_mcp — MCP retry delays (500ms+) exceed threshold. + # v28.0: :adjust_budget — GenServer.call(parent, ..., :infinity) blocks. + # v39.0: :answer_engine, :fetch_web, :call_api, :generate_images — HTTP + # API calls routinely exceed 100ms (DNS + TLS + server processing). execute_opts = - if action_atom == :call_mcp do - Keyword.put_new(execute_opts, :timeout, 600_000) - else - execute_opts + case action_atom do + :call_mcp -> Keyword.put_new(execute_opts, :timeout, 600_000) + :adjust_budget -> Keyword.put_new(execute_opts, :timeout, :infinity) + :answer_engine -> Keyword.put_new(execute_opts, :timeout, 120_000) + :fetch_web -> Keyword.put_new(execute_opts, :timeout, 60_000) + :call_api -> Keyword.put_new(execute_opts, :timeout, 120_000) + :generate_images -> Keyword.put_new(execute_opts, :timeout, 300_000) + _ -> execute_opts end # v25.0: Route check_id through existing Router from shell_routers, @@ -346,4 +360,8 @@ defmodule Quoracle.Agent.ConsensusHandler.ActionExecutor do state = Map.put(state, :active_routers, Map.put(active_routers, monitor_ref, router_pid)) {router_pid, state} end + + defp force_persist?(state) do + state |> Map.get(:test_opts, []) |> Keyword.get(:force_persist, false) + end end diff --git a/lib/quoracle/agent/consensus_handler/log_helper.ex b/lib/quoracle/agent/consensus_handler/log_helper.ex index 1a86907..8ec262f 100644 --- a/lib/quoracle/agent/consensus_handler/log_helper.ex +++ b/lib/quoracle/agent/consensus_handler/log_helper.ex @@ -23,7 +23,12 @@ defmodule Quoracle.Agent.ConsensusHandler.LogHelper do :missing_required_param, :invalid_param, :unknown_parameter, - :service_unavailable + :unknown_action, + :service_unavailable, + :connection_refused, + :connection_closed, + :timeout, + :econnrefused ] @doc """ @@ -34,6 +39,16 @@ defmodule Quoracle.Agent.ConsensusHandler.LogHelper do Logger.warning("Action failed: #{inspect(reason)}") end + # L5: Unwrap {:error, reason} wrapper (from ActionResultHandler) + def log_action_error({:error, reason}) do + log_action_error(reason) + end + + # L4: Handle {:action_crashed, tuple_reason} (noproc, timeout, etc.) + def log_action_error({:action_crashed, reason}) when is_tuple(reason) do + Logger.error("Action execution crashed: {:action_crashed, #{inspect(reason)}}") + end + # Registry errors during test cleanup are warnings, not errors def log_action_error({:action_crashed, msg}) when is_binary(msg) do if String.contains?(msg, "registry") do diff --git a/lib/quoracle/agent/core.ex b/lib/quoracle/agent/core.ex index 19f653e..3228cea 100644 --- a/lib/quoracle/agent/core.ex +++ b/lib/quoracle/agent/core.ex @@ -41,29 +41,23 @@ defmodule Quoracle.Agent.Core do @spec start_link(map() | {pid(), String.t()} | {pid(), String.t(), keyword()}, keyword()) :: GenServer.on_start() - # Production interface - uses global registry def start_link(config) do start_link(config, []) end - # Dependency injection interface - accepts registry and dynsup options def start_link(config, opts) when is_list(config) and is_list(opts) do - # Handle keyword list config - convert to map and proceed GenServer.start_link(__MODULE__, {Map.new(config), opts}) end def start_link(%{} = config, opts) do - # Pass options through to init GenServer.start_link(__MODULE__, {config, opts}) end def start_link({parent_pid, initial_prompt}, opts) do - # Handle tuple config from tests with options GenServer.start_link(__MODULE__, {{parent_pid, initial_prompt}, opts}) end def start_link({parent_pid, initial_prompt, test_opts}, opts) do - # Handle tuple config with test options from tests GenServer.start_link(__MODULE__, {{parent_pid, initial_prompt, test_opts}, opts}) end @@ -340,10 +334,10 @@ defmodule Quoracle.Agent.Core do BudgetHandler.handle_set_budget_allocated(new_budget, state) end - # Skills system (v27.0) - append learned skills to active_skills + # Skills system (v27.0), v38.0: invalidate cached prompt on skill changes def handle_cast({:learn_skills, skills_metadata}, state) when is_list(skills_metadata) do updated_skills = state.active_skills ++ skills_metadata - {:noreply, %{state | active_skills: updated_skills}} + {:noreply, invalidate_cached_system_prompt(%{state | active_skills: updated_skills})} end # Message and action handling - delegated to CastHandler (v24.0) @@ -464,7 +458,10 @@ defmodule Quoracle.Agent.Core do end end - Persistence.persist_ace_state(state) + # Skip persistence in test mode for speed (unless force_persist is set) + unless Map.get(state, :test_mode, false) && !force_persist?(state) do + Persistence.persist_ace_state(state) + end try do AgentEvents.broadcast_agent_terminated(state.agent_id, reason, state.pubsub) @@ -490,4 +487,11 @@ defmodule Quoracle.Agent.Core do router_pid -> {:reply, GenServer.call(router_pid, message), state} end end + + defp force_persist?(state) do + state |> Map.get(:test_opts, []) |> Keyword.get(:force_persist, false) + end + + # v38.0: Reset cached system prompt; called when prompt-affecting state changes. + defp invalidate_cached_system_prompt(state), do: %{state | cached_system_prompt: nil} end diff --git a/lib/quoracle/agent/core/AGENTS.md b/lib/quoracle/agent/core/AGENTS.md index 741b6b7..0a94c56 100644 --- a/lib/quoracle/agent/core/AGENTS.md +++ b/lib/quoracle/agent/core/AGENTS.md @@ -5,7 +5,8 @@ - Initialization: Init and DB setup (154 lines), start_link opts normalization - Persistence: DB persistence (149 lines), extract_parent_agent_id with state.parent_id fallback (v36.0), delegates ACE to submodule - Persistence.ACEState: ACE state serialization (332 lines), context_lessons + model_states -- MessageInfoHandler: Info message dispatch (329 lines), handle_trigger_consensus/1, handle_down/4, handle_spawn_failed/2 +- CastHandler: GenServer cast handling (150 lines), handle_store_mcp_client/2 (v38.0: Process.monitor on MCP client PID) +- MessageInfoHandler: Info message dispatch (333 lines), handle_trigger_consensus/1, handle_down/4, handle_spawn_failed/2 - TodoHandler: Per-agent task list management (57 lines) - BudgetHandler: Budget GenServer callbacks (247 lines), adjust_child_budget/4 (v37.0: cast-based, no child calls), handle_set_budget_allocated/2, release_child_budget/3 - ChildrenTracker: Children state management (63 lines), handle_child_spawned/2, handle_child_dismissed/2 @@ -20,10 +21,11 @@ ## MessageInfoHandler Key Handlers - handle_trigger_consensus/1: Unified consensus trigger with staleness check -- handle_down/4: Cleans up active_routers (by ref) and shell_routers (by PID scan) on Router death +- handle_down/4: Cleans up active_routers (by ref) and shell_routers (by PID scan) on Router death, clears mcp_client on MCP Client death (v38.0) - handle_spawn_failed/2: Logs warning, records failure in history, removes child, schedules consensus ## Patterns - Router lifecycle coupling: Core.terminate/2 stops Router via active_routers with :infinity timeout - Two-layer safety: Core stops Router explicitly + Router monitors Core as backup +- MCP Client lifecycle: Core monitors MCP Client via Process.monitor, clears state.mcp_client to nil on DOWN (v38.0) - Sandbox.allow in handle_continue (not init/1) to avoid race condition diff --git a/lib/quoracle/agent/core/budget_handler.ex b/lib/quoracle/agent/core/budget_handler.ex index 87b6260..a31fc53 100644 --- a/lib/quoracle/agent/core/budget_handler.ex +++ b/lib/quoracle/agent/core/budget_handler.ex @@ -214,7 +214,10 @@ defmodule Quoracle.Agent.Core.BudgetHandler do """ @spec handle_set_budget_allocated(Decimal.t(), State.t()) :: {:noreply, State.t()} def handle_set_budget_allocated(new_budget, state) do - new_budget_data = %{state.budget_data | allocated: new_budget} + budget_data = state.budget_data + # Ensure committed is a valid Decimal when transitioning from N/A mode + committed = budget_data[:committed] || Decimal.new(0) + new_budget_data = %{budget_data | allocated: new_budget, committed: committed} new_state = %{state | budget_data: new_budget_data} new_state = update_over_budget_status(new_state) {:noreply, new_state} diff --git a/lib/quoracle/agent/core/cast_handler.ex b/lib/quoracle/agent/core/cast_handler.ex index da3cec4..fbf587b 100644 --- a/lib/quoracle/agent/core/cast_handler.ex +++ b/lib/quoracle/agent/core/cast_handler.ex @@ -111,6 +111,7 @@ defmodule Quoracle.Agent.Core.CastHandler do """ @spec handle_store_mcp_client(pid(), State.t()) :: {:noreply, State.t()} def handle_store_mcp_client(mcp_client_pid, state) do + Process.monitor(mcp_client_pid) {:noreply, %{state | mcp_client: mcp_client_pid}} end diff --git a/lib/quoracle/agent/core/initialization.ex b/lib/quoracle/agent/core/initialization.ex index f982739..5403399 100644 --- a/lib/quoracle/agent/core/initialization.ex +++ b/lib/quoracle/agent/core/initialization.ex @@ -80,43 +80,49 @@ defmodule Quoracle.Agent.Core.Initialization do # Auto-detect restarts: Check if agent already exists in DB # This handles both supervisor restarts (crashes) AND manual restore_agent calls # Skip DB check entirely if already in restoration_mode (restore_agent set it explicitly) + # Skip DB entirely in test_mode unless force_persist is set (Option D optimization) state = if Map.get(state, :restoration_mode, false) do # Already flagged as restoration - skip DB check state else - # Defensive DB check - handle ownership errors gracefully - try do - case Quoracle.Tasks.TaskManager.get_agent(state.agent_id) do - {:ok, db_agent} -> - # DB record exists - this is a restart or restore - Logger.info("Agent #{state.agent_id} restarting, restoring state from DB") - - # Restore ACE state (context_lessons, model_states, model_histories) from DB - ace_state = Persistence.restore_ace_state(db_agent) - - %State{ + if Map.get(state, :test_mode, false) && !force_persist?(state) do + # Test mode: skip DB round-trips for faster tests + state + else + # Defensive DB check - handle ownership errors gracefully + try do + case Quoracle.Tasks.TaskManager.get_agent(state.agent_id) do + {:ok, db_agent} -> + # DB record exists - this is a restart or restore + Logger.info("Agent #{state.agent_id} restarting, restoring state from DB") + + # Restore ACE state (context_lessons, model_states, model_histories) from DB + ace_state = Persistence.restore_ace_state(db_agent) + + %State{ + state + | model_histories: ace_state.model_histories, + context_lessons: ace_state.context_lessons, + model_states: ace_state.model_states, + restoration_mode: true + } + + {:error, :not_found} -> + # Fresh spawn - persist to DB + Persistence.persist_agent(state) state - | model_histories: ace_state.model_histories, - context_lessons: ace_state.context_lessons, - model_states: ace_state.model_states, - restoration_mode: true - } - - {:error, :not_found} -> - # Fresh spawn - persist to DB - Persistence.persist_agent(state) + end + rescue + e in [DBConnection.OwnershipError, DBConnection.ConnectionError] -> + # DB not accessible - either no sandbox_owner OR owner exited during query + # Skip persistence entirely - agent will try again on next operation + Logger.debug( + "Skipping DB check for agent #{state.agent_id}: #{inspect(e.__struct__)}" + ) + state end - rescue - e in [DBConnection.OwnershipError, DBConnection.ConnectionError] -> - # DB not accessible - either no sandbox_owner OR owner exited during query - # Skip persistence entirely - agent will try again on next operation - Logger.debug( - "Skipping DB check for agent #{state.agent_id}: #{inspect(e.__struct__)}" - ) - - state end end @@ -159,4 +165,8 @@ defmodule Quoracle.Agent.Core.Initialization do {:noreply, state} end + + defp force_persist?(state) do + state |> Map.get(:test_opts, []) |> Keyword.get(:force_persist, false) + end end diff --git a/lib/quoracle/agent/core/message_info_handler.ex b/lib/quoracle/agent/core/message_info_handler.ex index c103916..339453a 100644 --- a/lib/quoracle/agent/core/message_info_handler.ex +++ b/lib/quoracle/agent/core/message_info_handler.ex @@ -292,6 +292,10 @@ defmodule Quoracle.Agent.Core.MessageInfoHandler do new_state = %{state | children: List.delete(state.children, pid)} {:noreply, new_state} + pid == state.mcp_client -> + Logger.info("Agent #{state.agent_id} MCP client terminated: #{inspect(reason)}") + {:noreply, %{state | mcp_client: nil}} + true -> {:noreply, state} end diff --git a/lib/quoracle/agent/core/state.ex b/lib/quoracle/agent/core/state.ex index 75a1898..de84fae 100644 --- a/lib/quoracle/agent/core/state.ex +++ b/lib/quoracle/agent/core/state.ex @@ -150,7 +150,9 @@ defmodule Quoracle.Agent.Core.State do # active_routers: monitor_ref => router_pid (all spawned Routers) active_routers: %{}, # shell_routers: command_id => router_pid (shell command Routers for status routing) - shell_routers: %{} + shell_routers: %{}, + # v38.0: Cached system prompt for consensus (lazy build, invalidated by learn_skills) + cached_system_prompt: nil ] @type t :: %__MODULE__{ @@ -232,7 +234,9 @@ defmodule Quoracle.Agent.Core.State do active_skills: [skill_metadata()], # v30.0: Per-action Router lifecycle active_routers: %{reference() => pid()}, - shell_routers: %{String.t() => pid()} + shell_routers: %{String.t() => pid()}, + # v38.0: Cached system prompt + cached_system_prompt: String.t() | nil } @type queued_message :: %{ @@ -414,7 +418,9 @@ defmodule Quoracle.Agent.Core.State do active_skills: Map.get(config, :active_skills, []), # v30.0: Per-action Router lifecycle active_routers: Map.get(config, :active_routers, %{}), - shell_routers: Map.get(config, :shell_routers, %{}) + shell_routers: Map.get(config, :shell_routers, %{}), + # v38.0: Cached system prompt + cached_system_prompt: Map.get(config, :cached_system_prompt) } end diff --git a/lib/quoracle/agent/dyn_sup.ex b/lib/quoracle/agent/dyn_sup.ex index 1fb9f70..7daa93c 100644 --- a/lib/quoracle/agent/dyn_sup.ex +++ b/lib/quoracle/agent/dyn_sup.ex @@ -261,6 +261,14 @@ defmodule Quoracle.Agent.DynSup do |> Map.put(:test_mode, true) end + # Pass through force_persist from opts (for tests that verify DB state) + config = + if Keyword.get(opts, :force_persist, false) do + Map.put(config, :force_persist, true) + else + config + end + # Pass through registry and pubsub from opts start_agent(dynsup_pid, config, opts) end diff --git a/lib/quoracle/agent/message_handler.ex b/lib/quoracle/agent/message_handler.ex index 64699e0..bdb7fa9 100644 --- a/lib/quoracle/agent/message_handler.ex +++ b/lib/quoracle/agent/message_handler.ex @@ -306,6 +306,30 @@ defmodule Quoracle.Agent.MessageHandler do # R10-R13: execute_consensus_action now always returns state (defaults wait: false) # Kept as helper because it's passed as function reference (&execute_consensus_action/2) + # v39.0: Fast-path actions skip ActionExecutor entirely (no Router/Task spawn, no continuation loop). + # Emits broadcasts inline so tests using {:action_started, _} as sync signals still work. + defp execute_consensus_action(state, %{_fast_path: true} = decision) do + state = StateUtils.add_history_entry(state, :decision, decision) + counter = Map.get(state, :action_counter, 0) + action_id = "action_#{state.agent_id}_#{counter + 1}" + state = Map.update(state, :action_counter, 1, &(&1 + 1)) + + pubsub = Map.get(state, :pubsub) + + AgentEvents.broadcast_action_started( + state.agent_id, + decision.action, + action_id, + decision.params, + pubsub + ) + + result = {:ok, decision.params} + AgentEvents.broadcast_action_completed(state.agent_id, action_id, result, pubsub) + + state + end + defp execute_consensus_action(state, decision) do ConsensusHandler.execute_consensus_action(state, decision, self()) end diff --git a/lib/quoracle/agent/message_handler/AGENTS.md b/lib/quoracle/agent/message_handler/AGENTS.md index fec696b..3dc7889 100644 --- a/lib/quoracle/agent/message_handler/AGENTS.md +++ b/lib/quoracle/agent/message_handler/AGENTS.md @@ -1,7 +1,7 @@ # lib/quoracle/agent/message_handler/ ## Modules -- ActionResultHandler: Action result processing (382 lines, extracted from MessageHandler 2026-02-13, v25.0 shell_routers + error-aware continuation, v27.0 consensus deferral) +- ActionResultHandler: Action result processing (383 lines, extracted from MessageHandler 2026-02-13, v25.0 shell_routers + error-aware continuation, v27.0 consensus deferral, v28.0 LogHelper.log_action_error wiring for server-side error logging) ## Key Functions - handle_action_result/4: Process action results with extended wait parameter handling. Routes through process_action_result → store in history → maybe_track_child → maybe_update_budget_committed → maybe_track_shell_router → flush_queued_messages → handle_action_result_continuation @@ -32,4 +32,5 @@ ## Dependencies - StateUtils: cancel_wait_timer/1, schedule_consensus_continuation/1, add_history_entry_with_action/4 - ConsensusHandler: handle_wait_parameter/3 (for timed waits) +- ConsensusHandler.LogHelper: log_action_error/1 (v28.0, server-side action error logging) - ImageDetector: detect/2 (for image result routing) diff --git a/lib/quoracle/agent/message_handler/action_result_handler.ex b/lib/quoracle/agent/message_handler/action_result_handler.ex index 58b2df6..19db53d 100644 --- a/lib/quoracle/agent/message_handler/action_result_handler.ex +++ b/lib/quoracle/agent/message_handler/action_result_handler.ex @@ -127,6 +127,7 @@ defmodule Quoracle.Agent.MessageHandler.ActionResultHandler do AgentEvents.broadcast_action_completed(state.agent_id, action_id, success, pubsub) {:error, _} = error -> + LogHelper.log_action_error(error) AgentEvents.broadcast_action_error(state.agent_id, action_id, error, pubsub) _ -> diff --git a/lib/quoracle/agent/reflector.ex b/lib/quoracle/agent/reflector.ex index 05c2365..a65040c 100644 --- a/lib/quoracle/agent/reflector.ex +++ b/lib/quoracle/agent/reflector.ex @@ -330,7 +330,16 @@ defmodule Quoracle.Agent.Reflector do # Handle ReqLLM.Response struct cond do is_struct(response, ReqLLM.Response) -> - ReqLLM.Response.text(response) + case ReqLLM.Response.text(response) do + text when is_binary(text) and text != "" -> + text + + _ -> + # Fallback: ReqLLM's build_content_parts detects pure JSON responses + # and stores them as %{type: :object} instead of %{type: :text}. + # Response.text() is blind to :object parts, so re-serialize. + if response.object, do: Jason.encode!(response.object), else: "" + end is_map(response) and is_binary(Map.get(response, :text)) -> Map.get(response, :text) diff --git a/lib/quoracle/agent/state_utils.ex b/lib/quoracle/agent/state_utils.ex index 372d6a5..2aa587f 100644 --- a/lib/quoracle/agent/state_utils.ex +++ b/lib/quoracle/agent/state_utils.ex @@ -168,9 +168,10 @@ defmodule Quoracle.Agent.StateUtils do """ @spec merge_consensus_state(map(), map()) :: map() def merge_consensus_state(genserver_state, consensus_state) do - ace_keys = [:model_histories, :context_lessons, :model_states] + # v38.0: Include cached_system_prompt so it persists after consensus + merge_keys = [:model_histories, :context_lessons, :model_states, :cached_system_prompt] - Enum.reduce(ace_keys, genserver_state, fn key, acc -> + Enum.reduce(merge_keys, genserver_state, fn key, acc -> if Map.has_key?(acc, key) and Map.has_key?(consensus_state, key) do Map.put(acc, key, Map.get(consensus_state, key)) else diff --git a/lib/quoracle/consensus/action_parser.ex b/lib/quoracle/consensus/action_parser.ex index da66179..3327778 100644 --- a/lib/quoracle/consensus/action_parser.ex +++ b/lib/quoracle/consensus/action_parser.ex @@ -127,7 +127,18 @@ defmodule Quoracle.Consensus.ActionParser do # Parse a single response into action format defp parse_single_response(%ReqLLM.Response{} = response, opts) do - content = ReqLLM.Response.text(response) + content = + case ReqLLM.Response.text(response) do + text when is_binary(text) and text != "" -> + text + + _ -> + # Fallback: ReqLLM's build_content_parts detects pure JSON responses + # and stores them as %{type: :object} instead of %{type: :text}. + # Response.text() is blind to :object parts, so re-serialize. + if response.object, do: Jason.encode!(response.object), else: nil + end + model = response.model parse_content(content, model, opts) end diff --git a/lib/quoracle/costs/AGENTS.md b/lib/quoracle/costs/AGENTS.md index 9a8d96e..2ba4a2d 100644 --- a/lib/quoracle/costs/AGENTS.md +++ b/lib/quoracle/costs/AGENTS.md @@ -3,7 +3,7 @@ ## Modules - AgentCost: Ecto schema for cost records (53 lines) - Recorder: Cost recording + PubSub broadcast (90 lines) -- Aggregator: Query module with recursive CTE for agent trees (396 lines) +- Aggregator: Query module with recursive CTE for agent trees (446 lines, v3.0) ## Key Functions - AgentCost.changeset/2: Validates cost_type in [llm_consensus, llm_embedding, llm_answer, llm_summarization, llm_condensation, image_generation, external, child_budget_absorbed] @@ -15,8 +15,13 @@ - Aggregator.by_task_and_model/1: JSONB aggregation by model_spec - Aggregator.by_task_and_model_detailed/1: v2.0 - 5 token types + aggregate costs (detailed breakdown) - Aggregator.by_agent_and_model_detailed/1: v2.0 - Same detailed breakdown for single agent +- Aggregator.by_agent_tree_and_model_detailed/1: v3.0 - Subtree costs including NULL model_spec (for DismissChild) +- Aggregator.by_agent_ids_and_model_detailed/1: v3.0 - Multi-agent detailed breakdown - Aggregator.get_descendant_agent_ids/1: Recursive CTE for agent tree +## Types +- model_cost_detailed: `model_spec: String.t() | nil` (nil for non-model costs like external/child_budget_absorbed) + ## Schema ``` agent_costs: id(uuid), agent_id(string), task_id(uuid), cost_type(string), @@ -29,10 +34,12 @@ agent_costs: id(uuid), agent_id(string), task_id(uuid), cost_type(string), - Decimal for monetary precision - Recursive CTE for agent tree traversal - JSONB for model_spec aggregation +- @detailed_select/@detailed_group_order module attributes (DRY SQL across detailed queries) +- execute_detailed_sql/2 shared helper for query execution + row mapping ## Dependencies - Ecto.Repo for DB operations - Phoenix.PubSub for broadcasts - TABLE_Tasks (foreign key) -Test coverage: 107 tests (29 schema + 28 recorder + 50 aggregator) +Test coverage: 111 tests (29 schema + 28 recorder + 54 aggregator) diff --git a/lib/quoracle/costs/aggregator.ex b/lib/quoracle/costs/aggregator.ex index 48c426a..a2902db 100644 --- a/lib/quoracle/costs/aggregator.ex +++ b/lib/quoracle/costs/aggregator.ex @@ -27,7 +27,7 @@ defmodule Quoracle.Costs.Aggregator do } @type model_cost_detailed :: %{ - model_spec: String.t(), + model_spec: String.t() | nil, request_count: non_neg_integer(), # Token counts (5 types) input_tokens: non_neg_integer(), @@ -269,6 +269,27 @@ defmodule Quoracle.Costs.Aggregator do # Detailed Per-Model Queries (v2.0 - Token Breakdown) # ============================================================ + # Shared SELECT clause for detailed model queries (DRY across single-ID and multi-ID variants) + @detailed_select """ + SELECT + metadata->>'model_spec' as model_spec, + COUNT(*) as request_count, + SUM(COALESCE((metadata->>'input_tokens')::integer, 0)) as input_tokens, + SUM(COALESCE((metadata->>'output_tokens')::integer, 0)) as output_tokens, + SUM(COALESCE((metadata->>'reasoning_tokens')::integer, 0)) as reasoning_tokens, + SUM(COALESCE((metadata->>'cached_tokens')::integer, 0)) as cached_tokens, + SUM(COALESCE((metadata->>'cache_creation_tokens')::integer, 0)) as cache_creation_tokens, + SUM(COALESCE((metadata->>'input_cost')::numeric, 0)) as input_cost, + SUM(COALESCE((metadata->>'output_cost')::numeric, 0)) as output_cost, + SUM(cost_usd) as total_cost + FROM agent_costs + """ + + @detailed_group_order """ + GROUP BY metadata->>'model_spec' + ORDER BY total_cost DESC NULLS LAST + """ + @doc """ Returns detailed costs grouped by model_spec for a task. Includes all 5 token types and aggregate costs. @@ -288,29 +309,21 @@ defmodule Quoracle.Costs.Aggregator do execute_detailed_model_query("agent_id", agent_id) end - # Shared SQL execution for detailed model queries + # Single-ID detailed query (filters out NULL model_spec for UI display) @spec execute_detailed_model_query(String.t(), binary() | String.t()) :: [model_cost_detailed()] defp execute_detailed_model_query(id_column, id_value) do - sql = """ - SELECT - metadata->>'model_spec' as model_spec, - COUNT(*) as request_count, - SUM(COALESCE((metadata->>'input_tokens')::integer, 0)) as input_tokens, - SUM(COALESCE((metadata->>'output_tokens')::integer, 0)) as output_tokens, - SUM(COALESCE((metadata->>'reasoning_tokens')::integer, 0)) as reasoning_tokens, - SUM(COALESCE((metadata->>'cached_tokens')::integer, 0)) as cached_tokens, - SUM(COALESCE((metadata->>'cache_creation_tokens')::integer, 0)) as cache_creation_tokens, - SUM(COALESCE((metadata->>'input_cost')::numeric, 0)) as input_cost, - SUM(COALESCE((metadata->>'output_cost')::numeric, 0)) as output_cost, - SUM(cost_usd) as total_cost - FROM agent_costs - WHERE #{id_column} = $1 - AND metadata->>'model_spec' IS NOT NULL - GROUP BY metadata->>'model_spec' - ORDER BY total_cost DESC NULLS LAST - """ + sql = + @detailed_select <> + "WHERE #{id_column} = $1\n AND metadata->>'model_spec' IS NOT NULL\n" <> + @detailed_group_order - case Repo.query(sql, [id_value]) do + execute_detailed_sql(sql, [id_value]) + end + + # Shared SQL execution for detailed model queries + @spec execute_detailed_sql(String.t(), [term()]) :: [model_cost_detailed()] + defp execute_detailed_sql(sql, params) do + case Repo.query(sql, params) do {:ok, %{rows: rows}} -> Enum.map(rows, &row_to_detailed_model_cost/1) @@ -360,6 +373,43 @@ defmodule Quoracle.Costs.Aggregator do defp to_decimal_or_nil(_), do: nil + # ============================================================ + # Per-Model Subtree Queries (v3.0) + # ============================================================ + + @doc """ + Returns detailed costs grouped by model_spec for an agent's entire subtree + (the agent itself plus all descendants). Includes costs with NULL model_spec + as a separate group (model_spec: nil in the returned map). + + Used by DismissChild to snapshot per-model costs before TreeTerminator + deletes cost records, enabling per-model absorption record creation. + """ + @spec by_agent_tree_and_model_detailed(String.t()) :: [model_cost_detailed()] + def by_agent_tree_and_model_detailed(agent_id) do + descendant_ids = get_descendant_agent_ids(agent_id) + all_ids = [agent_id | descendant_ids] + by_agent_ids_and_model_detailed(all_ids) + end + + @doc """ + Returns detailed costs grouped by model_spec for a list of agent_ids. + Includes records with NULL model_spec (returned with model_spec: nil). + """ + @spec by_agent_ids_and_model_detailed([String.t()]) :: [model_cost_detailed()] + def by_agent_ids_and_model_detailed([]), do: [] + + def by_agent_ids_and_model_detailed(agent_ids) when is_list(agent_ids) do + # Unlike execute_detailed_model_query, this includes NULL model_spec groups + # (needed by DismissChild to capture all costs including non-model ones) + sql = + @detailed_select <> + "WHERE agent_id = ANY($1)\n" <> + @detailed_group_order + + execute_detailed_sql(sql, [agent_ids]) + end + # ============================================================ # Individual Request Queries (for LogView) # ============================================================ diff --git a/lib/quoracle/mcp/AGENTS.md b/lib/quoracle/mcp/AGENTS.md index 206af90..13f0175 100644 --- a/lib/quoracle/mcp/AGENTS.md +++ b/lib/quoracle/mcp/AGENTS.md @@ -2,7 +2,7 @@ ## Modules -- **Client**: Per-agent MCP connection manager (427 lines) +- **Client**: Per-agent MCP connection manager (434 lines) - GenServer managing connections to MCP servers (stdio/HTTP) - Connection deduplication by command/url - Agent lifecycle monitoring with automatic cleanup @@ -10,6 +10,7 @@ - **v2.0**: Error context capture on initialization timeout - **v3.0**: Crash reason propagation via DOWN message capture - **v4.0**: Reconnect API, connection dead status, timeout 120s, crash protection in call_tool + - **v5.0**: Reason-aware logging in terminate/2 (info for normal/shutdown, warning for abnormal) - **ConnectionManager**: Connection lifecycle extracted from Client (~316 lines) - start_and_list_tools/4: Spawn anubis client, poll capabilities, list tools diff --git a/lib/quoracle/mcp/client.ex b/lib/quoracle/mcp/client.ex index 3168e9d..582fd86 100644 --- a/lib/quoracle/mcp/client.ex +++ b/lib/quoracle/mcp/client.ex @@ -355,7 +355,14 @@ defmodule Quoracle.MCP.Client do end @impl true - def terminate(_reason, state) do + def terminate(reason, state) do + case reason do + :normal -> Logger.info("MCP Client #{state.agent_id} terminating normally") + :shutdown -> Logger.info("MCP Client #{state.agent_id} shutting down") + {:shutdown, _} -> Logger.info("MCP Client #{state.agent_id} shutting down") + _ -> Logger.warning("MCP Client #{state.agent_id} terminating: #{inspect(reason)}") + end + cleanup_all_connections(state) :ok end diff --git a/lib/quoracle/providers/retry_helper.ex b/lib/quoracle/providers/retry_helper.ex index 160c1c0..4d06514 100644 --- a/lib/quoracle/providers/retry_helper.ex +++ b/lib/quoracle/providers/retry_helper.ex @@ -9,6 +9,8 @@ defmodule Quoracle.Providers.RetryHelper do v3.1: Exception handling for malformed LLM responses (empty body crashes from req_llm). v3.2: Detect HTTP 429/5xx from FunctionClauseError stacktrace args and retry (req_llm providers crash on error response bodies instead of returning proper errors). + v3.3: Handle string error bodies from Google Vertex throttling + (body is {"error":"The request is throttled..."} - string message, not {"error":{"code":429}}). """ require Logger @@ -189,8 +191,34 @@ defmodule Quoracle.Providers.RetryHelper do code end + # v3.3: Error map with string message: %{"error" => "The request is throttled..."} + # Google Vertex returns this format for throttling instead of structured error codes + defp find_error_code_in_args([%{"error" => message} | _]) when is_binary(message) do + infer_status_from_message(message) + end + + # v3.3: Undecoded string body (Req may skip JSON decode on some error responses) + defp find_error_code_in_args([response_body | _]) when is_binary(response_body) do + infer_status_from_message(response_body) + end + defp find_error_code_in_args(_), do: nil + # Infer HTTP status code from error message keywords + @spec infer_status_from_message(String.t()) :: pos_integer() | nil + defp infer_status_from_message(message) do + lowered = String.downcase(message) + + cond do + String.contains?(lowered, "throttl") -> 429 + String.contains?(lowered, "rate limit") -> 429 + String.contains?(lowered, "too many") -> 429 + String.contains?(lowered, "overloaded") -> 503 + String.contains?(lowered, "unavailable") -> 503 + true -> nil + end + end + @doc """ Applies exponential backoff delay. diff --git a/lib/quoracle/tasks/task_manager.ex b/lib/quoracle/tasks/task_manager.ex index 36339da..a52fd75 100644 --- a/lib/quoracle/tasks/task_manager.ex +++ b/lib/quoracle/tasks/task_manager.ex @@ -189,6 +189,7 @@ defmodule Quoracle.Tasks.TaskManager do |> maybe_put(:pubsub, pubsub) |> maybe_put(:force_init_error, Keyword.get(opts, :force_init_error)) |> maybe_put(:test_opts, Keyword.get(opts, :test_opts)) + |> maybe_put(:force_persist, Keyword.get(opts, :force_persist)) case AgentDynSup.start_agent(dynsup_pid, agent_config) do {:ok, root_pid} -> @@ -212,7 +213,7 @@ defmodule Quoracle.Tasks.TaskManager do # Build budget_data for root agent from task.budget_limit defp build_budget_data(nil) do - %{mode: :na, allocated: nil, committed: nil} + Quoracle.Budget.Schema.new_na() end defp build_budget_data(%Decimal{} = budget_limit) do diff --git a/lib/quoracle_web/live/ui/AGENTS.md b/lib/quoracle_web/live/ui/AGENTS.md index 2737a32..495051a 100644 --- a/lib/quoracle_web/live/ui/AGENTS.md +++ b/lib/quoracle_web/live/ui/AGENTS.md @@ -10,8 +10,9 @@ - LogEntry.Helpers: Formatting helpers (279 lines, 23 @spec), timestamp/level/metadata/role styling, LLM response formatting (2025-12) - Message: Accordion display, collapsed 80-char preview, reply forms with agent_alive control - CostDisplay: Cost display component (385 lines), 4 modes (:badge, :summary, :detail, :request) - - v2.0: Token breakdown table with 10 columns (Model, Req, Input, Output, Reason, Cache R, Cache W, In$, Out$, Total$) + - v3.0: Token breakdown table with 10 columns (Model, Req, Input, Output, Reason, Cache R, Cache W, In$, Out$, Total$) - Expandable detail view, lazy-loads from Aggregator + - Correctly renders absorption records (child_budget_absorbed with model_spec) after child dismissal - Helper functions: format_tokens/1, format_token_or_dash/1, format_cost_compact/1, truncate_model/1 ## Stateful Components @@ -70,5 +71,5 @@ - AgentNode: 28 tests + 6 direct message tests = 34 total (then refactored to 44 tests) - LogEntry: 25 tests, Message: 34 tests - Dashboard integration: 18 tests + 4 direct message tests = 22 total (then 35 tests) -- CostDisplay: 69 tests (v2.0 token breakdown table) +- CostDisplay: 72 tests (v3.0: R40-R42 absorption record acceptance tests) - BudgetUI Acceptance: 19 tests (R1-R19) - full E2E from /dashboard route (2025-12) diff --git a/mix.exs b/mix.exs index 1b1a71a..6fd300e 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Quoracle.MixProject do def project do [ app: :quoracle, - version: "0.1.15", + version: "0.1.16", elixir: "~> 1.18", listeners: [Phoenix.CodeReloader], elixirc_paths: elixirc_paths(Mix.env()), diff --git a/test/quoracle/actions/answer_engine_test.exs b/test/quoracle/actions/answer_engine_test.exs index 923c0e6..447be0e 100644 --- a/test/quoracle/actions/answer_engine_test.exs +++ b/test/quoracle/actions/answer_engine_test.exs @@ -109,23 +109,6 @@ defmodule Quoracle.Actions.AnswerEngineTest do end end - describe "[UNIT] timing metrics" do - test "includes execution timing in result (R8)" do - # R8: WHEN action executes IF successful THEN includes execution_time_ms - # This would be internal to execute, but we can test the structure - result = %{ - answer: "Test answer", - sources: [], - execution_time_ms: 150, - model: "google_gemini_2_5_pro" - } - - assert Map.has_key?(result, :execution_time_ms) - assert is_integer(result.execution_time_ms) - assert result.execution_time_ms > 0 - end - end - # NOTE: Old "[UNIT] Gemini model discovery" test removed (v3.0) # find_gemini_model() replaced by config-driven model selection # See "[INTEGRATION] config-driven model selection" tests below @@ -414,28 +397,6 @@ defmodule Quoracle.Actions.AnswerEngineTest do end end - describe "[INTEGRATION] credential and model management" do - test "uses ModelQuery to find Gemini model" do - # Verify integration with ModelQuery - # This would normally query the database - models = Quoracle.Models.ModelQuery.get_models_by_provider("google") - - # In test, might be empty or mocked - assert is_list(models) - - gemini_model = - Enum.find(models, fn model -> - model.model_id |> to_string() |> String.contains?("gemini") - end) - - # Gemini model may or may not exist in test DB - just verify query works - if gemini_model do - assert is_map(gemini_model) - assert gemini_model.model_id |> to_string() |> String.contains?("gemini") - end - end - end - # ============================================================= # CONFIG-DRIVEN MODEL SELECTION (v3.0 - feat-20251205-054538) # ============================================================= @@ -514,32 +475,5 @@ defmodule Quoracle.Actions.AnswerEngineTest do # Model used should be the configured one (model_spec from credential) assert response.model_used == "google-vertex:gemini-2.5-pro" end - - test "does not fall back to find_gemini_model when config missing (R3-config)" do - # Verify that when CONFIG_ModelSettings has no answer_engine_model, - # the action raises instead of falling back to find_gemini_model() - Quoracle.Models.TableConsensusConfig - |> Quoracle.Repo.delete_all() - - # Even if there are Google credentials in the DB, should raise - service_account_json = - Jason.encode!(%{"type" => "service_account", "project_id" => "test-project"}) - - unique_fallback = "google-vertex:gemini-fallback-#{System.unique_integer([:positive])}" - - {:ok, _cred} = - Quoracle.Models.TableCredentials.insert(%{ - model_id: unique_fallback, - model_spec: "google-vertex:gemini-2.5-pro", - api_key: service_account_json, - resource_id: "test-project", - region: "us-central1" - }) - - # Should raise RuntimeError, not use the fallback credential - assert_raise RuntimeError, ~r/Answer engine model not configured/, fn -> - AnswerEngine.execute(%{prompt: "Test"}, "agent-123", []) - end - end end end diff --git a/test/quoracle/actions/consensus_rules_batch_test.exs b/test/quoracle/actions/consensus_rules_batch_test.exs index 9785dbd..8cbda78 100644 --- a/test/quoracle/actions/consensus_rules_batch_test.exs +++ b/test/quoracle/actions/consensus_rules_batch_test.exs @@ -5,7 +5,7 @@ defmodule Quoracle.Actions.ConsensusRulesBatchTest do Packet: 1 (Schema Foundation) """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Actions.ConsensusRules # ARC Verification Criteria from ACTION_ConsensusRules v8.0 @@ -169,23 +169,6 @@ defmodule Quoracle.Actions.ConsensusRulesBatchTest do assert Enum.at(merged, 0).action == :todo end - test "handles string keys from LLM responses" do - # LLMs may return string keys instead of atom keys - sequences = [ - [ - %{"action" => :file_read, "params" => %{path: "/a.txt"}} - ], - [ - %{"action" => :file_read, "params" => %{path: "/a.txt"}} - ] - ] - - # Should handle both atom and string keys - result = ConsensusRules.apply_rule(:batch_sequence_merge, sequences) - # May need normalization - verify it doesn't crash - assert is_tuple(result) - end - test "merges with multiple param keys per action" do sequences = [ [ diff --git a/test/quoracle/actions/consensus_rules_test.exs b/test/quoracle/actions/consensus_rules_test.exs index 3833b3a..2dc4438 100644 --- a/test/quoracle/actions/consensus_rules_test.exs +++ b/test/quoracle/actions/consensus_rules_test.exs @@ -63,13 +63,6 @@ defmodule Quoracle.Actions.ConsensusRulesTest do assert {:ok, ""} = ConsensusRules.apply_rule(rule, values) end - test "handles caching for repeated identical strings" do - # All identical - triggers optimization, no API call - values = ["Query database", "Query database", "Query database"] - rule = {:semantic_similarity, threshold: 0.85} - assert {:ok, "Query database"} = ConsensusRules.apply_rule(rule, values) - end - test "returns no_consensus when embeddings unavailable for different strings" do # Different strings need embedding API - without credentials returns error values = ["Process data", "Beautiful sunset", "Pizza recipe"] diff --git a/test/quoracle/actions/dismiss_child_budget_test.exs b/test/quoracle/actions/dismiss_child_budget_test.exs index db0a8db..c7b50a9 100644 --- a/test/quoracle/actions/dismiss_child_budget_test.exs +++ b/test/quoracle/actions/dismiss_child_budget_test.exs @@ -239,7 +239,7 @@ defmodule Quoracle.Actions.DismissChildBudgetTest do {:ok, parent_pid} = spawn_agent_with_budget("parent-R20", deps, parent_budget) # Create child with N/A budget (no allocation) - child_budget = %{mode: :na, allocated: nil, committed: nil} + child_budget = Quoracle.Budget.Schema.new_na() {:ok, _child_pid} = spawn_agent_with_budget("child-R20", deps, child_budget, diff --git a/test/quoracle/actions/dismiss_child_reconciliation_test.exs b/test/quoracle/actions/dismiss_child_reconciliation_test.exs index d3aa487..95c656b 100644 --- a/test/quoracle/actions/dismiss_child_reconciliation_test.exs +++ b/test/quoracle/actions/dismiss_child_reconciliation_test.exs @@ -1,9 +1,9 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do @moduledoc """ - Tests for ACTION_DismissChild v4.0 - Budget Reconciliation on Dismissal. + Tests for ACTION_DismissChild v4.0/v5.0 - Budget Reconciliation on Dismissal. - WorkGroupID: fix-20260211-budget-enforcement - Packet: Packet 2 (Dismissal Reconciliation) + WorkGroupID: fix-20260211-budget-enforcement, fix-20260223-cost-display-budget-timeout + Packet: Packet 2 (Dismissal Reconciliation), Packet 1 (Per-Model Cost Absorption) Tests the corrected budget reconciliation flow when parent dismisses child: - R22: Tree spent queried before termination @@ -16,15 +16,29 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do - R28b: N/A child with costs creates absorption record - R29: Over budget re-evaluated after absorption - R30: Acceptance - full dismissal budget flow + + v5.0 Requirements (fix-20260223-cost-display-budget-timeout): + - R31: Per-model absorption records created + - R32: Model spec preserved in absorption metadata + - R33: Token counts preserved in absorption metadata + - R34: Non-model costs absorbed without model_spec + - R35: Cost Detail model table preserves totals (SYSTEM) + - R36: Absorption succeeds when parent dead + - R37: No Core.get_state call (uses parent_id directly) + - R38: Escrow still released when parent alive + - R39: Escrow skipped when parent dead, costs still absorbed + - R40: Property - absorption records sum equals tree spent + - R41: Property - task total unchanged after dismissal + - R42: Escrow release before absorption record creation (audit gap) """ use Quoracle.DataCase, async: true + use ExUnitProperties alias Quoracle.Actions.DismissChild alias Quoracle.Agent.Core alias Quoracle.Costs.AgentCost - # Aggregator used for understanding query patterns in test assertions - # (not directly called in tests, but documented for context) + alias Quoracle.Costs.Aggregator alias Quoracle.Tasks.Task, as: TaskSchema alias Test.IsolationHelpers @@ -111,6 +125,40 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do cost end + # Helper to insert a cost record with model_spec and token metadata (v5.0) + defp insert_model_cost(agent_id, task_id, opts) do + cost_usd = Keyword.fetch!(opts, :cost_usd) + model_spec = Keyword.get(opts, :model_spec) + cost_type = Keyword.get(opts, :cost_type, "llm_consensus") + + metadata = + %{ + "input_tokens" => Keyword.get(opts, :input_tokens, 500), + "output_tokens" => Keyword.get(opts, :output_tokens, 200), + "reasoning_tokens" => Keyword.get(opts, :reasoning_tokens, 0), + "cached_tokens" => Keyword.get(opts, :cached_tokens, 0), + "cache_creation_tokens" => Keyword.get(opts, :cache_creation_tokens, 0), + "input_cost" => Keyword.get(opts, :input_cost, "0.01"), + "output_cost" => Keyword.get(opts, :output_cost, "0.02") + } + |> then(fn m -> + if model_spec, do: Map.put(m, "model_spec", model_spec), else: m + end) + + {:ok, cost} = + Repo.insert( + AgentCost.changeset(%AgentCost{}, %{ + agent_id: agent_id, + task_id: task_id, + cost_type: cost_type, + cost_usd: cost_usd, + metadata: metadata + }) + ) + + cost + end + # ========================================================================== # R22: Tree Spent Queried Before Termination [INTEGRATION] # ========================================================================== @@ -487,7 +535,7 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do } # Child with N/A budget (no allocation) - child_budget = %{mode: :na, allocated: nil, committed: nil} + child_budget = Quoracle.Budget.Schema.new_na() {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) @@ -532,7 +580,7 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do committed: Decimal.new("0") } - child_budget = %{mode: :na, allocated: nil, committed: nil} + child_budget = Quoracle.Budget.Schema.new_na() {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) @@ -776,4 +824,896 @@ defmodule Quoracle.Actions.DismissChildReconciliationTest do assert absorption.metadata["unspent_returned"] == "20.00" end end + + # ========================================================================== + # v5.0: R31 - Per-Model Records Created [INTEGRATION] + # ========================================================================== + + describe "per-model absorption (R31)" do + @tag :r31 + @tag :integration + test "R31: creates per-model absorption records on dismissal", + %{deps: deps, task: task} do + parent_id = "parent-R31-#{System.unique_integer([:positive])}" + child_id = "child-R31-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + # Child has costs across 2 different models + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("15.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("10.00"), + model_spec: "openai/gpt-4o" + ) + + # Act: Dismiss child + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: 2 absorption records created, one per model + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed", + order_by: [desc: c.cost_usd] + ) + ) + + assert length(absorption_records) == 2, + "Should create 2 absorption records (one per model), " <> + "got #{length(absorption_records)}" + + # Verify each record has correct cost + costs = Enum.map(absorption_records, & &1.cost_usd) + assert Enum.any?(costs, &Decimal.equal?(&1, Decimal.new("15.00"))) + assert Enum.any?(costs, &Decimal.equal?(&1, Decimal.new("10.00"))) + end + end + + # ========================================================================== + # v5.0: R32 - Model Spec Preserved in Metadata [INTEGRATION] + # ========================================================================== + + describe "model_spec metadata (R32)" do + @tag :r32 + @tag :integration + test "R32: absorption record metadata preserves model_spec", + %{deps: deps, task: task} do + parent_id = "parent-R32-#{System.unique_integer([:positive])}" + child_id = "child-R32-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Act + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption metadata includes model_spec as string key + [absorption] = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + assert absorption.metadata["model_spec"] == "anthropic/claude-sonnet-4", + "Absorption metadata must preserve model_spec. " <> + "Got: #{inspect(absorption.metadata)}" + end + end + + # ========================================================================== + # v5.0: R33 - Token Counts Preserved [INTEGRATION] + # ========================================================================== + + describe "token preservation (R33)" do + @tag :r33 + @tag :integration + test "R33: absorption records preserve token counts and costs", + %{deps: deps, task: task} do + parent_id = "parent-R33-#{System.unique_integer([:positive])}" + child_id = "child-R33-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4", + input_tokens: 1000, + output_tokens: 500, + reasoning_tokens: 200, + cached_tokens: 100, + cache_creation_tokens: 50, + input_cost: "0.05", + output_cost: "0.10" + ) + + # Act + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption metadata includes all 5 token types and costs + [absorption] = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + meta = absorption.metadata + + assert meta["input_tokens"] == "1000" + assert meta["output_tokens"] == "500" + assert meta["reasoning_tokens"] == "200" + assert meta["cached_tokens"] == "100" + assert meta["cache_creation_tokens"] == "50" + assert meta["input_cost"] != nil + assert meta["output_cost"] != nil + end + end + + # ========================================================================== + # v5.0: R34 - Non-Model Costs Absorbed [INTEGRATION] + # ========================================================================== + + describe "non-model cost absorption (R34)" do + @tag :r34 + @tag :integration + test "R34: external costs absorbed without model_spec", + %{deps: deps, task: task} do + parent_id = "parent-R34-#{System.unique_integer([:positive])}" + child_id = "child-R34-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + # Child has external cost (no model_spec) + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("5.00"), + cost_type: "external" + ) + + # Act + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption record created without model_spec + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + assert length(absorption_records) == 1 + + absorption = hd(absorption_records) + assert Decimal.equal?(absorption.cost_usd, Decimal.new("5.00")) + + # External costs should NOT have model_spec in metadata + refute Map.has_key?(absorption.metadata, "model_spec"), + "External cost absorption should not have model_spec" + end + end + + # ========================================================================== + # v5.0: R35 - Model Table Total Preserved [SYSTEM] + # ========================================================================== + + describe "model table total (R35)" do + @tag :r35 + @tag :acceptance + @tag :system + test "R35: model table total unchanged after child dismissal", + %{deps: deps, task: task} do + parent_id = "parent-R35-#{System.unique_integer([:positive])}" + child_id = "child-R35-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("200.00"), + committed: Decimal.new("100.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("100.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + # Parent has own costs on model A + insert_model_cost(parent_id, task.id, + cost_usd: Decimal.new("10.00"), + model_spec: "anthropic/claude-sonnet-4", + input_tokens: 500, + output_tokens: 200 + ) + + # Child has costs across 2 models + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("25.00"), + model_spec: "anthropic/claude-sonnet-4", + input_tokens: 1200, + output_tokens: 400 + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("15.00"), + model_spec: "openai/gpt-4o", + input_tokens: 800, + output_tokens: 300 + ) + + # BEFORE dismissal: snapshot per-model breakdown and header total + before_task_total = Aggregator.by_task(task.id).total_cost + before_model_detail = Aggregator.by_task_and_model_detailed(task.id) + + before_model_sum = + Enum.reduce(before_model_detail, Decimal.new("0"), fn row, acc -> + if row.total_cost, do: Decimal.add(acc, row.total_cost), else: acc + end) + + # Sanity: model sum should equal header total before dismissal + assert Decimal.equal?(before_model_sum, before_task_total), + "Before dismissal: model sum #{before_model_sum} must equal " <> + "header total #{before_task_total}" + + # Act: Parent dismisses child + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # AFTER dismissal: header total unchanged + after_task_total = Aggregator.by_task(task.id).total_cost + + assert Decimal.equal?(after_task_total, before_task_total), + "Header total must not change after dismissal. " <> + "Before: #{before_task_total}, After: #{after_task_total}" + + # AFTER dismissal: per-model sum still equals header total + after_model_detail = Aggregator.by_task_and_model_detailed(task.id) + + after_model_sum = + Enum.reduce(after_model_detail, Decimal.new("0"), fn row, acc -> + if row.total_cost, do: Decimal.add(acc, row.total_cost), else: acc + end) + + assert Decimal.equal?(after_model_sum, after_task_total), + "After dismissal: model sum #{after_model_sum} must equal " <> + "header total #{after_task_total}. " <> + "Models: #{inspect(Enum.map(after_model_detail, & &1.model_spec))}" + + # AFTER dismissal: both models still visible with correct attribution + claude_after = + Enum.find(after_model_detail, &(&1.model_spec == "anthropic/claude-sonnet-4")) + + gpt_after = Enum.find(after_model_detail, &(&1.model_spec == "openai/gpt-4o")) + + assert claude_after != nil, + "Claude model must still be visible after dismissal" + + assert gpt_after != nil, + "GPT model must still be visible after dismissal" + end + end + + # ========================================================================== + # v5.0: R36 - Absorption When Parent Dead [INTEGRATION] + # ========================================================================== + + describe "dead parent absorption (R36)" do + @tag :r36 + @tag :integration + test "R36: absorption records created even when parent process dead", + %{deps: deps, task: task} do + parent_id = "parent-R36-#{System.unique_integer([:positive])}" + child_id = "child-R36-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Kill parent before dismissal reconciliation + GenServer.stop(parent_pid, :normal, :infinity) + + # Act: Dismiss child (parent is dead) + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption records still created in DB despite parent being dead + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + assert absorption_records != [], + "Absorption records must be created even when parent process is dead" + + total_absorbed = + absorption_records + |> Enum.map(& &1.cost_usd) + |> Enum.reject(&is_nil/1) + |> Enum.reduce(Decimal.new("0"), &Decimal.add/2) + + assert Decimal.equal?(total_absorbed, Decimal.new("20.00")), + "Total absorbed should be $20.00 regardless of parent liveness" + end + end + + # ========================================================================== + # v5.0: R37 - No Core.get_state Call [UNIT] + # ========================================================================== + + describe "parent_id direct usage (R37)" do + @tag :r37 + @tag :unit + test "R37: absorption uses parent_id directly, no GenServer call", + %{deps: deps, task: task} do + parent_id = "parent-R37-#{System.unique_integer([:positive])}" + child_id = "child-R37-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Kill parent so Core.get_state would fail if called + GenServer.stop(parent_pid, :normal, :infinity) + + # Act: Dismiss child with dead parent + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption record created with correct parent agent_id + # This proves parent_id string was used directly, not Core.get_state + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + assert absorption_records != [], + "Absorption must succeed using parent_id string directly, " <> + "without calling Core.get_state on dead parent process" + + # Verify the agent_id on the record matches parent_id + Enum.each(absorption_records, fn record -> + assert record.agent_id == parent_id, + "Absorption record agent_id must be parent_id (#{parent_id})" + end) + end + end + + # ========================================================================== + # v5.0: R38 - Escrow Still Released [INTEGRATION] + # ========================================================================== + + describe "escrow release (R38)" do + @tag :r38 + @tag :integration + test "R38: escrow release still works for live parent", + %{deps: deps, task: task} do + parent_id = "parent-R38-#{System.unique_integer([:positive])}" + child_id = "child-R38-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Act + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Parent's committed decreased (escrow released) + {:ok, parent_state} = Core.get_state(parent_pid) + + assert Decimal.equal?(parent_state.budget_data.committed, Decimal.new("0")), + "Parent committed should decrease from 50 to 0 after escrow release" + end + end + + # ========================================================================== + # v5.0: R39 - Escrow Skipped, Costs Absorbed [INTEGRATION] + # ========================================================================== + + describe "dead parent escrow skip (R39)" do + @tag :r39 + @tag :integration + test "R39: escrow skipped for dead parent, costs still absorbed", + %{deps: deps, task: task} do + parent_id = "parent-R39-#{System.unique_integer([:positive])}" + child_id = "child-R39-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Kill parent to make escrow release impossible + GenServer.stop(parent_pid, :normal, :infinity) + + # Act + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Absorption records created even though escrow couldn't be released + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + assert absorption_records != [], + "Costs must still be absorbed even when escrow release skipped" + end + end + + # ========================================================================== + # v5.0: R42 - Escrow Release Before Absorption Records [INTEGRATION] + # + # Integration audit found critical race condition: if parent crashes between + # absorption record creation and escrow release, budget leaks (committed + # amount never returned). Fix: escrow release MUST happen BEFORE absorption + # record creation. Escrow is the volatile operation (needs live parent); + # absorption is durable (pure DB insert, no process needed). + # + # Test strategy: Suspend parent GenServer BEFORE dismiss starts. The background + # task will block on whichever operation first requires the parent GenServer: + # - Current (absorption first): Absorption records created (pure DB), then + # Core.release_child_budget blocks on suspended parent. Records exist in DB. + # - Fixed (escrow first): Core.release_child_budget blocks immediately on + # suspended parent. No absorption records created yet. + # + # Synchronization: poll_until child cost records are deleted (TreeTerminator + # completed), then poll_until absorption records appear OR a stabilization + # window expires (proving reconciliation's first operation is blocking). + # + # Assert no absorption records while parent is suspended → proves escrow + # was attempted first (blocking), not absorption (non-blocking DB writes). + # ========================================================================== + + describe "escrow-first ordering (R42)" do + @tag :r42 + @tag :integration + test "R42: escrow release attempted before absorption record creation", + %{deps: deps, task: task} do + parent_id = "parent-R42-#{System.unique_integer([:positive])}" + child_id = "child-R42-#{System.unique_integer([:positive])}" + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("50.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("50.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + # Child has costs that will need absorption + insert_model_cost(child_id, task.id, + cost_usd: Decimal.new("20.00"), + model_spec: "anthropic/claude-sonnet-4" + ) + + # Verify the cost record exists before dismissal + child_costs_before = + Repo.aggregate( + from(c in AgentCost, where: c.agent_id == ^child_id), + :count + ) + + assert child_costs_before == 1 + + # Suspend the parent GenServer. This blocks any GenServer.call to it, + # including Core.release_child_budget. Pure DB operations (Recorder.record) + # are unaffected by the suspension. + :sys.suspend(parent_pid) + + # Trigger dismiss — background task starts + {:ok, _} = DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + + # EVENT-BASED SYNC: Poll until TreeTerminator has deleted the child's + # cost records. This proves TreeTerminator completed and the background + # task has moved on to reconcile_child_budget. + :ok = + IsolationHelpers.poll_until( + fn -> + Repo.aggregate( + from(c in AgentCost, where: c.agent_id == ^child_id), + :count + ) == 0 + end, + 10_000 + ) + + # At this point, TreeTerminator is done and reconcile_child_budget has + # started (or is about to start). The first operation in reconcile will + # either complete instantly (DB insert) or block (GenServer.call to + # suspended parent). + # + # EVENT-BASED SYNC: Poll briefly for absorption records. If they appear, + # absorption happened first (current buggy ordering). If they don't appear + # within a reasonable window, the background task is blocked on the + # GenServer.call (correct escrow-first ordering). + # + # We poll for absorption records with a bounded timeout. With absorption- + # first ordering, records appear almost instantly after TreeTerminator + # finishes (pure DB insert, <50ms). With escrow-first ordering, the task + # is blocked on the suspended parent and records never appear. + absorption_appeared = + case IsolationHelpers.poll_until( + fn -> + Repo.aggregate( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ), + :count + ) > 0 + end, + 500 + ) do + :ok -> true + {:error, :timeout} -> false + end + + # KEY ASSERTION: With escrow-first ordering, Core.release_child_budget + # (GenServer.call to suspended parent) blocks BEFORE absorption records + # are created. Absorption records should NOT appear within 500ms. + # + # With current code (absorption first), absorption records are pure DB + # writes that complete instantly (~10ms), so they DO appear. + refute absorption_appeared, + "With escrow-first ordering, absorption records should NOT exist yet " <> + "while parent is suspended (escrow GenServer.call should block first). " <> + "Records appeared — this means absorption happened before escrow " <> + "release (wrong ordering, budget leak risk)." + + # Resume parent so the blocked GenServer.call completes + :sys.resume(parent_pid) + + # Now wait for the full dismiss to complete + :ok = wait_for_dismiss_complete(child_id) + + # After resuming, both operations should complete successfully + final_absorption_count = + Repo.aggregate( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ), + :count + ) + + assert final_absorption_count > 0, + "After resume, absorption records should be created" + + {:ok, parent_state} = Core.get_state(parent_pid) + + assert Decimal.equal?(parent_state.budget_data.committed, Decimal.new("0")), + "After resume, escrow should be released (committed = 0)" + end + end + + # ========================================================================== + # v5.0: R40 - Property: Absorption Sum Equals Tree Spent [UNIT] + # ========================================================================== + + describe "absorption sum property (R40)" do + @tag :r40 + @tag :property + property "absorption records sum equals tree spent total", + %{deps: deps, task: task} do + check all( + cost1_cents <- integer(1..5000), + cost2_cents <- integer(1..5000) + ) do + parent_id = "parent-R40-#{System.unique_integer([:positive])}" + child_id = "child-R40-#{System.unique_integer([:positive])}" + + cost1 = Decimal.div(Decimal.new(cost1_cents), 100) + cost2 = Decimal.div(Decimal.new(cost2_cents), 100) + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("10000.00"), + committed: Decimal.new("10000.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("10000.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + # Child has costs across 2 models + insert_model_cost(child_id, task.id, + cost_usd: cost1, + model_spec: "model/a" + ) + + insert_model_cost(child_id, task.id, + cost_usd: cost2, + model_spec: "model/b" + ) + + expected_total = Decimal.add(cost1, cost2) + + # Act + {:ok, _} = + DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Sum of absorption records equals tree spent + absorption_records = + Repo.all( + from(c in AgentCost, + where: c.agent_id == ^parent_id and c.cost_type == "child_budget_absorbed" + ) + ) + + absorption_sum = + absorption_records + |> Enum.map(& &1.cost_usd) + |> Enum.reject(&is_nil/1) + |> Enum.reduce(Decimal.new("0"), &Decimal.add/2) + + assert Decimal.equal?(absorption_sum, expected_total), + "Absorption sum #{absorption_sum} must equal tree spent #{expected_total}" + end + end + end + + # ========================================================================== + # v5.0: R41 - Property: Task Total Unchanged [INTEGRATION] + # ========================================================================== + + describe "task total invariant (R41)" do + @tag :r41 + @tag :property + property "task total cost unchanged by dismissal", + %{deps: deps, task: task} do + check all(child_cost_cents <- integer(1..5000)) do + parent_id = "parent-R41-#{System.unique_integer([:positive])}" + child_id = "child-R41-#{System.unique_integer([:positive])}" + + child_cost = Decimal.div(Decimal.new(child_cost_cents), 100) + + parent_budget = %{ + mode: :root, + allocated: Decimal.new("10000.00"), + committed: Decimal.new("10000.00") + } + + child_budget = %{ + mode: :allocated, + allocated: Decimal.new("10000.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_agent_with_budget(parent_id, deps, task, parent_budget) + + {:ok, _child_pid} = + spawn_agent_with_budget(child_id, deps, task, child_budget, + parent_id: parent_id, + parent_pid: parent_pid + ) + + insert_model_cost(child_id, task.id, + cost_usd: child_cost, + model_spec: "anthropic/claude-sonnet-4" + ) + + # Snapshot task total BEFORE dismissal + task_total_before = Aggregator.by_task(task.id).total_cost + + # Act + {:ok, _} = + DismissChild.execute(%{child_id: child_id}, parent_id, action_opts(deps, task)) + + :ok = wait_for_dismiss_complete(child_id) + + # Assert: Task total unchanged + task_total_after = Aggregator.by_task(task.id).total_cost + + assert Decimal.equal?( + task_total_after || Decimal.new("0"), + task_total_before || Decimal.new("0") + ), + "Task total must not change after dismissal. " <> + "Before: #{task_total_before}, After: #{task_total_after}" + end + end + end end diff --git a/test/quoracle/actions/mcp_test.exs b/test/quoracle/actions/mcp_test.exs index aa36a4b..99b18c8 100644 --- a/test/quoracle/actions/mcp_test.exs +++ b/test/quoracle/actions/mcp_test.exs @@ -135,26 +135,6 @@ defmodule Quoracle.Actions.MCPTest do assert length(tools) == 2 assert Enum.any?(tools, &(&1.name == "read_file")) end - - test "connect returns connection_id and tools", %{opts: opts, agent_id: agent_id} do - anubis_pid = placeholder_process() - - expect(Quoracle.MCP.AnubisMock, :start_link, fn _opts -> - {:ok, anubis_pid} - end) - - expect(Quoracle.MCP.AnubisMock, :list_tools, fn ^anubis_pid -> - {:ok, mock_tools()} - end) - - params = %{transport: :stdio, command: "npx @mcp/server"} - - result = MCP.execute(params, agent_id, opts) - - assert {:ok, %{connection_id: conn_id, tools: tools}} = result - assert is_binary(conn_id) - assert is_list(tools) - end end # ============================================================================ @@ -460,21 +440,6 @@ defmodule Quoracle.Actions.MCPTest do assert {:error, :xor_violation} = result end - - test "http transport with connection_id returns xor_violation", %{ - opts: opts, - agent_id: agent_id - } do - params = %{ - transport: :http, - url: "https://example.com", - connection_id: "some-connection" - } - - result = MCP.execute(params, agent_id, opts) - - assert {:error, :xor_violation} = result - end end # ============================================================================ @@ -585,19 +550,6 @@ defmodule Quoracle.Actions.MCPTest do end end - # ============================================================================ - # R13: 3-Arity Signature - # [UNIT] WHEN execute called THEN accepts (params, agent_id, opts) signature - # ============================================================================ - describe "R13: 3-arity signature" do - test "follows standard 3-arity action signature", %{opts: opts, agent_id: agent_id} do - # Call with 3 arguments - compiler verifies function exists, - # test verifies it handles params correctly - result = MCP.execute(%{}, agent_id, opts) - assert {:error, :invalid_params} = result - end - end - # ============================================================================ # R14: Agent System Test # [SYSTEM] WHEN agent uses call_mcp via consensus THEN full flow works diff --git a/test/quoracle/actions/orient_enhanced_test.exs b/test/quoracle/actions/orient_enhanced_test.exs index b0073a8..55ae296 100644 --- a/test/quoracle/actions/orient_enhanced_test.exs +++ b/test/quoracle/actions/orient_enhanced_test.exs @@ -182,49 +182,6 @@ defmodule Quoracle.Actions.OrientEnhancedTest do end end - describe "integration with ACTION_Router" do - test "orient results are observable through router metrics", %{ - agent_id: agent_id, - pubsub: pubsub - } do - alias Quoracle.Actions.Router - - # Per-action Router (v28.0): Spawn Router for this specific action - action_id = "action-#{System.unique_integer([:positive])}" - - {:ok, router} = - Router.start_link( - action_type: :orient, - action_id: action_id, - agent_id: agent_id, - agent_pid: self(), - pubsub: pubsub, - sandbox_owner: nil - ) - - on_exit(fn -> - if Process.alive?(router), do: GenServer.stop(router, :normal, :infinity) - end) - - # Subscribe to both action events and agent logs on isolated PubSub - Phoenix.PubSub.subscribe(pubsub, "actions:all") - Phoenix.PubSub.subscribe(pubsub, "agents:#{agent_id}:logs") - - # Execute orient through router - {:ok, _} = Router.execute(router, :orient, valid_orient_params(), agent_id, pubsub: pubsub) - - # Should receive both router and orient broadcasts - assert_receive {:action_started, start_event}, 30_000 - assert start_event.action_type == :orient - - assert_receive {:log_entry, orient_event}, 30_000 - assert orient_event.metadata.action == "orient" - - assert_receive {:action_completed, complete_event}, 30_000 - assert match?({:ok, _}, complete_event.result) - end - end - # Helper functions defp valid_orient_params do %{ diff --git a/test/quoracle/actions/orient_isolation_test.exs b/test/quoracle/actions/orient_isolation_test.exs index 8718721..be33d7e 100644 --- a/test/quoracle/actions/orient_isolation_test.exs +++ b/test/quoracle/actions/orient_isolation_test.exs @@ -108,76 +108,7 @@ defmodule Quoracle.Actions.OrientIsolationTest do end end - describe "PubSub isolation for action events" do - test "broadcasts action events to isolated PubSub", %{pubsub: pubsub} do - agent_id = "test-orient-action-#{System.unique_integer([:positive])}" - - # Subscribe to action events in isolated PubSub - :ok = Phoenix.PubSub.subscribe(pubsub, "actions:all") - - # Spawn per-action Router (v28.0) - router = spawn_orient_router(agent_id, pubsub) - - on_exit(fn -> - if Process.alive?(router) do - try do - GenServer.stop(router, :normal, :infinity) - catch - :exit, _ -> :ok - end - end - end) - - # Execute orient action through router - capture_log(fn -> - send(self(), {:result, Router.execute(router, :orient, @valid_params, agent_id)}) - end) - - assert_received {:result, {:ok, _}} - - # Should receive action started event - assert_receive {:action_started, start_payload} - assert start_payload.agent_id == agent_id - assert start_payload.action_type == :orient - - # Should receive action completed event - assert_receive {:action_completed, complete_payload} - assert complete_payload.agent_id == agent_id - end - end - describe "error handling with isolation" do - test "broadcasts validation errors to isolated PubSub", %{pubsub: pubsub} do - agent_id = "test-orient-error" - - # Subscribe to action events - :ok = Phoenix.PubSub.subscribe(pubsub, "actions:all") - - # Spawn per-action Router (v28.0) - router = spawn_orient_router(agent_id, pubsub) - - on_exit(fn -> - if Process.alive?(router) do - try do - GenServer.stop(router, :normal, :infinity) - catch - :exit, _ -> :ok - end - end - end) - - # Invalid params (missing required fields) - capture_log(fn -> - send(self(), {:result, Router.execute(router, :orient, %{}, agent_id)}) - end) - - assert_received {:result, {:error, _validation_errors}} - - # Should receive error broadcast in isolated PubSub - assert_receive {:action_error, payload} - assert payload.agent_id == agent_id - end - test "broadcasts execution errors to isolated PubSub", %{pubsub: pubsub} do agent_id = "test-orient-exec-error" diff --git a/test/quoracle/actions/orient_validation_test.exs b/test/quoracle/actions/orient_validation_test.exs deleted file mode 100644 index 77449f2..0000000 --- a/test/quoracle/actions/orient_validation_test.exs +++ /dev/null @@ -1,124 +0,0 @@ -defmodule Quoracle.Actions.OrientValidationTest do - use ExUnit.Case, async: true - alias Quoracle.Actions.Validator - - describe "orient action validation with all optional params" do - test "validates orient action with all required and optional parameters" do - # This is the exact scenario reported by the user - action_json = %{ - "action" => "orient", - "params" => %{ - # Required params - "current_situation" => - "I have been given a high-level indication that I will be working on a significant task", - "goal_clarity" => - "Very low - I know the task will involve an AI-run business but have no specifics", - "available_resources" => - "I have access to various actions including web fetching, shell execution", - "key_challenges" => "Complete lack of specificity about the task requirements", - "delegation_consideration" => - "Cannot assess delegation needs without knowing task specifics", - # Optional params - these were causing the validation failure - "approach_options" => - "Wait for detailed instructions, then assess whether the task requires research", - "assumptions" => - "The task will be substantial and complex given the 'significant' qualifier", - "constraints_impact" => - "Cannot proceed with any meaningful work until more information is provided", - "next_steps" => "Wait for detailed task instructions and requirements from the user", - "parallelization_opportunities" => "Cannot determine without knowing task specifics", - "risk_factors" => - "Acting without sufficient information, misunderstanding requirements", - "success_criteria" => - "Unknown - will need to be defined once task details are provided", - "unknowns" => "Task scope, timeline, specific objectives, business domain" - } - } - - assert {:ok, validated} = Validator.validate_action(action_json) - assert validated.action == :orient - - # Verify all params were converted to atoms - assert is_atom(Map.keys(validated.params) |> List.first()) - - # Check specific optional params that were previously failing - assert validated.params.approach_options == - "Wait for detailed instructions, then assess whether the task requires research" - - assert validated.params.assumptions == - "The task will be substantial and complex given the 'significant' qualifier" - - assert validated.params.constraints_impact == - "Cannot proceed with any meaningful work until more information is provided" - - assert validated.params.unknowns == - "Task scope, timeline, specific objectives, business domain" - end - - test "validates orient with only required params" do - action_json = %{ - "action" => "orient", - "params" => %{ - "current_situation" => "Starting a new task", - "goal_clarity" => "High - objectives are clear", - "available_resources" => "Full action suite available", - "key_challenges" => "None identified", - "delegation_consideration" => "No delegation needed for this straightforward task" - } - } - - assert {:ok, validated} = Validator.validate_action(action_json) - assert validated.action == :orient - assert Map.keys(validated.params) |> length() == 5 - end - - test "validates orient with mix of required and some optional params" do - action_json = %{ - "action" => "orient", - "params" => %{ - "current_situation" => "Mid-task assessment", - "goal_clarity" => "Medium", - "available_resources" => "Limited to web actions", - "key_challenges" => "API rate limits", - "delegation_consideration" => "May need to delegate API work if rate limits persist", - "next_steps" => "Implement caching strategy", - "risk_factors" => "Potential for rate limit errors" - } - } - - assert {:ok, validated} = Validator.validate_action(action_json) - assert validated.action == :orient - assert validated.params.next_steps == "Implement caching strategy" - assert validated.params.risk_factors == "Potential for rate limit errors" - end - - test "fails validation when required orient param is missing" do - action_json = %{ - "action" => "orient", - "params" => %{ - "current_situation" => "Starting", - "goal_clarity" => "High" - # Missing: available_resources, key_challenges - } - } - - assert {:error, :missing_required_param} = Validator.validate_action(action_json) - end - - test "fails validation with truly unknown parameter" do - action_json = %{ - "action" => "orient", - "params" => %{ - "current_situation" => "Starting", - "goal_clarity" => "High", - "available_resources" => "All", - "key_challenges" => "None", - "delegation_consideration" => "None needed", - "completely_unknown_param" => "This should fail" - } - } - - assert {:error, :unknown_parameter} = Validator.validate_action(action_json) - end - end -end diff --git a/test/quoracle/actions/router_action_mapper_test.exs b/test/quoracle/actions/router_action_mapper_test.exs deleted file mode 100644 index a2afb9f..0000000 --- a/test/quoracle/actions/router_action_mapper_test.exs +++ /dev/null @@ -1,93 +0,0 @@ -defmodule Quoracle.Actions.Router.ActionMapperTest do - @moduledoc """ - Tests for Router's ActionMapper module. - Verifies that answer_engine is properly mapped. - """ - - use ExUnit.Case, async: true - alias Quoracle.Actions.Router.ActionMapper - - describe "answer_engine mapping" do - test "get_action_module/1 maps answer_engine to AnswerEngine module" do - # Verify that ActionMapper includes answer_engine mapping - result = ActionMapper.get_action_module(:answer_engine) - assert {:ok, Quoracle.Actions.AnswerEngine} = result - end - - test "returns ok tuple for valid answer_engine action" do - # Verify answer_engine is recognized as a valid action - result = ActionMapper.get_action_module(:answer_engine) - assert {:ok, _module} = result - end - - test "returns error for unmapped actions" do - # Verify unmapped actions return error - result = ActionMapper.get_action_module(:nonexistent_action) - assert {:error, :not_implemented} = result - end - end - - # ACTION_Router v18.0 - Search Secrets Action Routing - # ARC Verification Criteria for search_secrets action mapping - describe "search_secrets mapping (v18.0)" do - # R1: ActionMapper Includes search_secrets - test "ActionMapper routes search_secrets to SearchSecrets module" do - # [UNIT] - WHEN get_action_module(:search_secrets) called THEN returns {:ok, Quoracle.Actions.SearchSecrets} - result = ActionMapper.get_action_module(:search_secrets) - assert {:ok, Quoracle.Actions.SearchSecrets} = result - end - end - - # ACTION_Router v19.0 - Dismiss Child Action Routing - # ARC Verification Criteria for dismiss_child action mapping - # WorkGroupID: feat-20251224-dismiss-child - # Packet: 1 (Infrastructure) - describe "dismiss_child mapping (v19.0)" do - # R1: ActionMapper Includes dismiss_child - test "ActionMapper routes dismiss_child to DismissChild module" do - # [UNIT] - WHEN get_action_module(:dismiss_child) called THEN returns {:ok, Quoracle.Actions.DismissChild} - result = ActionMapper.get_action_module(:dismiss_child) - assert {:ok, Quoracle.Actions.DismissChild} = result - end - end - - # ACTION_Router v20.0 - generate_images Action Routing - # ARC Verification Criteria for generate_images action mapping - # WorkGroupID: feat-20251229-052855 - # Packet: 3 (Action Integration) - describe "generate_images mapping (v20.0)" do - # R1: ActionMapper Includes generate_images - test "ActionMapper routes generate_images to GenerateImages module" do - # [UNIT] - WHEN get_action_module(:generate_images) called THEN returns {:ok, Quoracle.Actions.GenerateImages} - result = ActionMapper.get_action_module(:generate_images) - assert {:ok, Quoracle.Actions.GenerateImages} = result - end - end - - # ACTION_Router v22.0 - adjust_budget Action Routing - # ARC Verification Criteria for adjust_budget action mapping - # WorkGroupID: feat-20251231-191717 - # Packet: 1 (Action Definition) - describe "adjust_budget mapping (v22.0)" do - # R10: ActionMapper Entry [UNIT] - test "R10: get_action_module returns AdjustBudget for adjust_budget" do - # [UNIT] - WHEN get_action_module(:adjust_budget) THEN returns AdjustBudget module - result = ActionMapper.get_action_module(:adjust_budget) - assert {:ok, Quoracle.Actions.AdjustBudget} = result - end - end - - # ACTION_Router v24.0 - batch_sync Action Routing - # Integration gap fix from feat-20260123-batch-sync audit - # WorkGroupID: feat-20260123-batch-sync - # Packet: 4 (Integration Fix) - describe "batch_sync mapping (v24.0)" do - # R1: ActionMapper Includes batch_sync - @tag :batch_sync - test "R1: get_action_module returns BatchSync for batch_sync" do - # [UNIT] - WHEN get_action_module(:batch_sync) THEN returns BatchSync module - result = ActionMapper.get_action_module(:batch_sync) - assert {:ok, Quoracle.Actions.BatchSync} = result - end - end -end diff --git a/test/quoracle/actions/router_batch_sync_test.exs b/test/quoracle/actions/router_batch_sync_test.exs index ba04f3d..c0191ae 100644 --- a/test/quoracle/actions/router_batch_sync_test.exs +++ b/test/quoracle/actions/router_batch_sync_test.exs @@ -234,5 +234,37 @@ defmodule Quoracle.Actions.RouterBatchSyncTest do assert :batch_sync in sync_actions, "batch_sync should be in always_sync_actions for synchronous execution" end + + # TEST-FIX: R63 regression guard — deferred from TEST phase because the current + # code already matches the spec (13 actions). This guards against accidental + # additions/removals from the canonical list. + test "always_sync_actions contains exactly the canonical 13 actions" do + # [UNIT] - R63: Canonical @always_sync_actions list per ACTION_Router v33.0 + sync_actions = ClientAPI.always_sync_actions() + + expected = + MapSet.new([ + :spawn_child, + :send_message, + :orient, + :wait, + :todo, + :generate_secret, + :adjust_budget, + :learn_skills, + :create_skill, + :search_secrets, + :file_read, + :file_write, + :batch_sync + ]) + + actual = MapSet.new(sync_actions) + + assert actual == expected, + "always_sync_actions mismatch.\n" <> + "Missing: #{inspect(MapSet.difference(expected, actual) |> MapSet.to_list())}\n" <> + "Extra: #{inspect(MapSet.difference(actual, expected) |> MapSet.to_list())}" + end end end diff --git a/test/quoracle/actions/router_generate_images_test.exs b/test/quoracle/actions/router_generate_images_test.exs index 81f61e2..28baecf 100644 --- a/test/quoracle/actions/router_generate_images_test.exs +++ b/test/quoracle/actions/router_generate_images_test.exs @@ -131,39 +131,18 @@ defmodule Quoracle.Actions.RouterGenerateImagesTest do action_id: action_id, sandbox_owner: sandbox_owner, plug: plug, - agent_pid: self() + agent_pid: self(), + # Force synchronous execution to prevent sandbox ownership loss in nested + # Task.async_stream workers under load (generate_images spawns parallel tasks) + timeout: 15_000 ] params = %{"prompt" => "A beautiful landscape"} - # Subscribe to action events for async result notification - Phoenix.PubSub.subscribe(pubsub, "actions:all") - # Should succeed for generalist (no permission check) result = Router.execute(router, :generate_images, params, agent_id, opts) - # Per-action Router (v28.0): await_result not supported, use PubSub for async results - response = - case result do - {:ok, resp} -> - resp - - {:async, _ref} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - after - 5000 -> flunk("No action completion message received") - end - - {:async, _ref, _ack} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - after - 5000 -> flunk("No action completion message received") - end - end + assert {:ok, response} = result # Must return generate_images result - proves no access control blocks generalists assert response.action == "generate_images" @@ -226,42 +205,17 @@ defmodule Quoracle.Actions.RouterGenerateImagesTest do action_id: action_id, sandbox_owner: sandbox_owner, plug: plug, - agent_pid: self() + agent_pid: self(), + # Force synchronous execution to prevent sandbox ownership loss in nested + # Task.async_stream workers under load (generate_images spawns parallel tasks) + timeout: 15_000 ] params = %{"prompt" => "Test prompt for routing"} - # Subscribe to action events for async result notification - Phoenix.PubSub.subscribe(pubsub, "actions:all") - - # Execute and verify it routes to GenerateImages module + # Should return synchronously with timeout opt result = Router.execute(router, :generate_images, params, agent_id, opts) - - # Per-action Router (v28.0): await_result not supported, use PubSub for async results - # Handle both success (action_completed) and error (action_error) broadcasts - response = - case result do - {:ok, resp} -> - resp - - {:async, _ref} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - {:action_error, %{error: error}} -> flunk("Action failed: #{inspect(error)}") - after - 10_000 -> flunk("No action completion message received within 10s") - end - - {:async, _ref, _ack} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - {:action_error, %{error: error}} -> flunk("Action failed: #{inspect(error)}") - after - 10_000 -> flunk("No action completion message received within 10s") - end - end + assert {:ok, response} = result # Should return image generation result with correct format assert %{action: "generate_images", images: images} = response @@ -320,40 +274,16 @@ defmodule Quoracle.Actions.RouterGenerateImagesTest do action_id: action_id, sandbox_owner: sandbox_owner, plug: plug, - agent_pid: self() + agent_pid: self(), + # Force synchronous execution to prevent sandbox ownership loss in nested + # Task.async_stream workers under load (generate_images spawns parallel tasks) + timeout: 15_000 ] params = %{"prompt" => "Generate a happy image"} - # Subscribe to action events for async result notification - Phoenix.PubSub.subscribe(pubsub, "actions:all") - result = Router.execute(router, :generate_images, params, agent_id, opts) - - # Per-action Router (v28.0): await_result not supported, use PubSub for async results - response = - case result do - {:ok, resp} -> - resp - - {:async, _ref} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - {:action_error, %{error: error}} -> {:error, error} - after - 5000 -> flunk("No action completion message received") - end - - {:async, _ref, _ack} -> - receive do - {:action_completed, %{result: {:ok, resp}}} -> resp - {:action_completed, %{result: resp}} -> resp - {:action_error, %{error: error}} -> {:error, error} - after - 5000 -> flunk("No action completion message received") - end - end + assert {:ok, response} = result assert response.action == "generate_images" assert is_list(response.images) @@ -401,45 +331,20 @@ defmodule Quoracle.Actions.RouterGenerateImagesTest do end end) - # Subscribe to action events for async result notification - Phoenix.PubSub.subscribe(pubsub, "actions:all") + opts = [ + registry: registry, + action_id: action_id, + agent_pid: self(), + # Force synchronous execution for deterministic test behavior + timeout: 5000 + ] - opts = [registry: registry, action_id: action_id, agent_pid: self()] params = %{"prompt" => "This should fail"} result = Router.execute(router, :generate_images, params, agent_id, opts) - # Per-action Router (v28.0): await_result not supported, use PubSub for async results - # Handle both sync and async result paths consistently - final_result = - case result do - {:error, _} = err -> - err - - {:async, _ref} -> - receive do - {:action_completed, %{result: async_result}} -> async_result - {:action_error, %{error: {:error, _} = err}} -> err - {:action_error, %{error: reason}} -> {:error, reason} - after - 5000 -> flunk("No action completion message received") - end - - {:async, _ref, _metadata} -> - receive do - {:action_completed, %{result: async_result}} -> async_result - {:action_error, %{error: {:error, _} = err}} -> err - {:action_error, %{error: reason}} -> {:error, reason} - after - 5000 -> flunk("No action completion message received") - end - - {:ok, resp} -> - {:ok, resp} - end - # Should return meaningful error about no models - assert final_result == {:error, :no_models_configured} + assert result == {:error, :no_models_configured} end # Parameter validation through Router @@ -476,33 +381,19 @@ defmodule Quoracle.Actions.RouterGenerateImagesTest do end end) - # Subscribe to action events for async result notification - Phoenix.PubSub.subscribe(pubsub, "actions:all") + opts = [ + registry: registry, + action_id: action_id, + agent_pid: self(), + # Force synchronous execution for deterministic test behavior + timeout: 5000 + ] - opts = [registry: registry, action_id: action_id, agent_pid: self()] params = %{} result = Router.execute(router, :generate_images, params, agent_id, opts) - # Per-action Router (v28.0): await_result not supported, use PubSub for async results - error = - case result do - {:error, reason} -> - reason - - {:async, _ref} -> - receive do - {:action_completed, %{result: {:error, reason}}} -> reason - {:action_error, %{error: reason}} -> reason - after - 5000 -> flunk("No action completion message received") - end - - {:ok, _} -> - :unexpected_success - end - - assert error == :missing_required_param + assert {:error, :missing_required_param} = result end end end diff --git a/test/quoracle/actions/router_isolation_test.exs b/test/quoracle/actions/router_isolation_test.exs deleted file mode 100644 index 953e982..0000000 --- a/test/quoracle/actions/router_isolation_test.exs +++ /dev/null @@ -1,114 +0,0 @@ -defmodule Quoracle.Actions.RouterIsolationTest do - @moduledoc """ - Tests for PubSub isolation in ACTION_Router. - Verifies that Router uses AgentEvents for broadcasts which support isolation. - """ - - use ExUnit.Case, async: true - import ExUnit.CaptureLog - alias Quoracle.Actions.Router - alias Test.PubSubIsolation - - setup do - # Setup isolated PubSub for this test - {:ok, pubsub} = PubSubIsolation.setup_isolated_pubsub() - - agent_id = "test-agent-#{System.unique_integer([:positive])}" - - # Per-action Router (v28.0) - use :orient for isolation tests - {:ok, router} = - Router.start_link( - action_type: :orient, - action_id: "action-#{System.unique_integer([:positive])}", - agent_id: agent_id, - agent_pid: self(), - pubsub: pubsub - ) - - # Ensure router terminates before test exits - on_exit(fn -> - if Process.alive?(router) do - try do - GenServer.stop(router, :normal, :infinity) - catch - :exit, _ -> :ok - end - end - end) - - # Subscribe to action events in isolated PubSub - :ok = Phoenix.PubSub.subscribe(pubsub, "actions:all") - - %{router: router, pubsub: pubsub, agent_id: agent_id} - end - - describe "PubSub isolation" do - test "broadcasts action_started events to isolated PubSub", %{router: router} do - # Execute an action that should broadcast - capture_log(fn -> - Router.execute(router, :wait, %{wait: 0.01}, "test-agent-123") - end) - - # Should receive broadcast in isolated PubSub - assert_receive {:action_started, payload} - assert payload.agent_id == "test-agent-123" - assert payload.action_type == :wait - end - - test "broadcasts action_completed events to isolated PubSub", %{router: router} do - # Execute a quick action that completes synchronously - capture_log(fn -> - Router.execute( - router, - :orient, - %{ - current_situation: "test", - goal_clarity: "clear", - available_resources: "test env", - key_challenges: "none", - delegation_consideration: "none" - }, - "test-agent-456" - ) - end) - - # Should receive completion broadcast in isolated PubSub - assert_receive {:action_completed, payload} - assert payload.agent_id == "test-agent-456" - end - - test "broadcasts action_error events to isolated PubSub", %{router: router} do - # Execute with invalid params to trigger error - capture_log(fn -> - Router.execute(router, :wait, %{wait: -1}, "test-agent-789") - end) - - # Should receive error broadcast in isolated PubSub - assert_receive {:action_error, payload} - assert payload.agent_id == "test-agent-789" - end - - test "isolated broadcasts don't leak between tests", %{router: router} do - # This test's broadcasts should be isolated - agent_id = "test-agent-main" - other_agent = "other-agent" - - # Set up another isolated PubSub instance - {:ok, other_pubsub} = PubSubIsolation.setup_isolated_pubsub() - - # Subscribe to the other PubSub instance - Phoenix.PubSub.subscribe(other_pubsub, "actions:all") - - # Execute with the main router (using this test's PubSub) - capture_log(fn -> - Router.execute(router, :wait, %{wait: 0.005}, agent_id) - end) - - # Should receive broadcast in this test's isolated PubSub - assert_receive {:action_started, %{agent_id: ^agent_id}} - - # Should NOT receive broadcasts from other tests - refute_receive {:action_started, %{agent_id: ^other_agent}}, 100 - end - end -end diff --git a/test/quoracle/actions/router_metadata_test.exs b/test/quoracle/actions/router_metadata_test.exs index 41c71ca..b16f9fb 100644 --- a/test/quoracle/actions/router_metadata_test.exs +++ b/test/quoracle/actions/router_metadata_test.exs @@ -155,54 +155,6 @@ defmodule Quoracle.Actions.RouterMetadataTest do end end - describe "backward compatibility for other actions" do - test "Orient action maintains its existing signature", %{ - router: router, - pubsub: pubsub, - agent_id: agent_id - } do - params = %{ - current_situation: "Testing", - goal_clarity: "Clear", - available_resources: "Adequate", - key_challenges: "None", - delegation_consideration: "None needed" - } - - opts = [ - pubsub: pubsub, - agent_pid: self() - ] - - # Orient should still work with its existing signature - result = Router.execute(router, :orient, params, agent_id, opts) - - # Should execute successfully - assert {:ok, _} = result - end - - test "Wait action maintains its existing signature", %{ - router: router, - registry: registry, - pubsub: pubsub, - agent_id: agent_id - } do - params = %{wait: 0} - - opts = [ - registry: registry, - pubsub: pubsub, - agent_pid: self() - ] - - # Wait should still work with its existing signature - result = Router.execute(router, :wait, params, agent_id, opts) - - # Should execute successfully - assert {:ok, _} = result - end - end - describe "error handling" do test "returns error when SendMessage module not loaded", %{ router: router, diff --git a/test/quoracle/actions/router_per_action_lifecycle_test.exs b/test/quoracle/actions/router_per_action_lifecycle_test.exs index f3011b5..a225f99 100644 --- a/test/quoracle/actions/router_per_action_lifecycle_test.exs +++ b/test/quoracle/actions/router_per_action_lifecycle_test.exs @@ -427,104 +427,6 @@ defmodule Quoracle.Actions.RouterPerActionLifecycleTest do end end - # ============================================================================= - # State Simplification Tests (R46-R48) - # ============================================================================= - - describe "Router state simplification (R46-R48)" do - setup do - pubsub = :"pubsub_#{System.unique_integer([:positive])}" - start_supervised!({Phoenix.PubSub, name: pubsub}) - {:ok, pubsub: pubsub, sandbox_owner: self()} - end - - @tag :r46 - test "R46: Router state has empty active_tasks map", %{ - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent_pid = self() - action_id = "action-#{System.unique_integer([:positive])}" - - {:ok, router_pid} = - Router.start_link( - action_type: :orient, - action_id: action_id, - agent_id: "test-agent", - agent_pid: agent_pid, - pubsub: pubsub, - sandbox_owner: sandbox_owner - ) - - on_exit(fn -> - if Process.alive?(router_pid), do: GenServer.stop(router_pid, :normal, :infinity) - end) - - state = :sys.get_state(router_pid) - - # Per-action Router has active_tasks for Execution.handle_task_completion - # but it's empty initially (no multi-action tracking needed) - assert state.active_tasks == %{} - end - - @tag :r47 - test "R47: Router state has empty results map", %{ - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent_pid = self() - action_id = "action-#{System.unique_integer([:positive])}" - - {:ok, router_pid} = - Router.start_link( - action_type: :orient, - action_id: action_id, - agent_id: "test-agent", - agent_pid: agent_pid, - pubsub: pubsub, - sandbox_owner: sandbox_owner - ) - - on_exit(fn -> - if Process.alive?(router_pid), do: GenServer.stop(router_pid, :normal, :infinity) - end) - - state = :sys.get_state(router_pid) - - # Per-action Router has results for Execution.handle_task_completion - # but it's empty initially (no multi-action storage needed) - assert state.results == %{} - end - - @tag :r48 - test "R48: Router state has no metrics aggregation", %{ - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent_pid = self() - action_id = "action-#{System.unique_integer([:positive])}" - - {:ok, router_pid} = - Router.start_link( - action_type: :orient, - action_id: action_id, - agent_id: "test-agent", - agent_pid: agent_pid, - pubsub: pubsub, - sandbox_owner: sandbox_owner - ) - - on_exit(fn -> - if Process.alive?(router_pid), do: GenServer.stop(router_pid, :normal, :infinity) - end) - - state = :sys.get_state(router_pid) - - # Per-action Router should NOT have metrics aggregation (ephemeral) - refute Map.has_key?(state, :metrics) - end - end - # ============================================================================= # Validation Still Works Tests (R49-R51) # ============================================================================= diff --git a/test/quoracle/actions/router_secret_gaps_test.exs b/test/quoracle/actions/router_secret_gaps_test.exs index a255c79..7825473 100644 --- a/test/quoracle/actions/router_secret_gaps_test.exs +++ b/test/quoracle/actions/router_secret_gaps_test.exs @@ -106,7 +106,7 @@ defmodule Quoracle.Actions.RouterSecretGapsTest do pubsub: pubsub, sandbox_owner: self(), agent_pid: self(), - timeout: 1000, + timeout: 5000, capability_groups: capability_groups ] @@ -142,7 +142,7 @@ defmodule Quoracle.Actions.RouterSecretGapsTest do sandbox_owner: self(), agent_pid: self(), task_id: "task-#{System.unique_integer([:positive])}", - timeout: 1000, + timeout: 5000, capability_groups: capability_groups ] @@ -183,7 +183,7 @@ defmodule Quoracle.Actions.RouterSecretGapsTest do pubsub: pubsub, sandbox_owner: self(), agent_pid: self(), - timeout: 1000, + timeout: 5000, capability_groups: capability_groups ] diff --git a/test/quoracle/actions/router_send_message_test.exs b/test/quoracle/actions/router_send_message_test.exs index d57335b..4e5d3a1 100644 --- a/test/quoracle/actions/router_send_message_test.exs +++ b/test/quoracle/actions/router_send_message_test.exs @@ -359,12 +359,4 @@ defmodule Quoracle.Actions.RouterSendMessageTest do assert response[:action] == "send_message" end end - - describe "Router integration with action priorities" do - test "send_message has correct priority in Schema", %{router: _router} do - # The Schema already defines send_message with priority 3 - # This test verifies it's accessible through the Router's validation - assert Quoracle.Actions.Schema.get_action_priority(:send_message) == 3 - end - end end diff --git a/test/quoracle/actions/router_skills_test.exs b/test/quoracle/actions/router_skills_test.exs index 8be0da3..fd94f99 100644 --- a/test/quoracle/actions/router_skills_test.exs +++ b/test/quoracle/actions/router_skills_test.exs @@ -11,33 +11,14 @@ defmodule Quoracle.Actions.RouterSkillsTest do WorkGroupID: feat-20260112-skills-system """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Actions.Router - alias Quoracle.Actions.Router.ActionMapper # ========================================================================== # R20-R21: ActionMapper Entries # ========================================================================== - describe "ActionMapper skill action mappings (R20-R21)" do - # R20: ActionMapper Includes learn_skills - test "ActionMapper routes learn_skills to LearnSkills module" do - result = ActionMapper.get_action_module(:learn_skills) - assert {:ok, Quoracle.Actions.LearnSkills} = result - end - - # R21: ActionMapper Includes create_skill - test "ActionMapper routes create_skill to CreateSkill module" do - result = ActionMapper.get_action_module(:create_skill) - assert {:ok, Quoracle.Actions.CreateSkill} = result - end - end - - # ========================================================================== - # R22: No Access Control - # ========================================================================== - describe "skill actions access control (R22)" do setup do # Create isolated test environment diff --git a/test/quoracle/actions/schema_adjust_budget_test.exs b/test/quoracle/actions/schema_adjust_budget_test.exs index e97e703..aaf5492 100644 --- a/test/quoracle/actions/schema_adjust_budget_test.exs +++ b/test/quoracle/actions/schema_adjust_budget_test.exs @@ -16,14 +16,6 @@ defmodule Quoracle.Actions.SchemaAdjustBudgetTest do assert :adjust_budget in actions end - # R2: Schema Definition [UNIT] - test "R2: get_schema returns adjust_budget schema" do - result = Schema.get_schema(:adjust_budget) - assert {:ok, schema} = result - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :param_types) - end - # R3: Required Parameters [UNIT] test "R3: adjust_budget requires child_id and new_budget" do {:ok, schema} = Schema.get_schema(:adjust_budget) @@ -47,13 +39,6 @@ defmodule Quoracle.Actions.SchemaAdjustBudgetTest do assert description =~ "WHEN" assert description =~ "budget" end - - # R6: Action Priority [UNIT] - test "R6: adjust_budget has priority" do - priority = Schema.get_action_priority(:adjust_budget) - assert is_integer(priority) - assert priority > 0 - end end describe "adjust_budget param_types" do diff --git a/test/quoracle/actions/schema_api_call_test.exs b/test/quoracle/actions/schema_api_call_test.exs index e73482c..37a4d2d 100644 --- a/test/quoracle/actions/schema_api_call_test.exs +++ b/test/quoracle/actions/schema_api_call_test.exs @@ -7,17 +7,6 @@ defmodule Quoracle.Actions.SchemaApiCallTest do # R1: WHEN list_actions called THEN includes :call_api in action list actions = Schema.list_actions() assert :call_api in actions - assert length(actions) == 22 - end - - test "retrieves call_api schema" do - # R2: WHEN get_schema(:call_api) called THEN returns call_api schema - assert {:ok, schema} = Schema.get_schema(:call_api) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - assert Map.has_key?(schema, :param_descriptions) end test "call_api has correct required parameters" do @@ -44,12 +33,6 @@ defmodule Quoracle.Actions.SchemaApiCallTest do assert description =~ "JSON-RPC" end - test "call_api has priority 17" do - # R6: WHEN get_action_priority(:call_api) called THEN returns 17 - priority = Schema.get_action_priority(:call_api) - assert priority == 17 - end - test "call_api has correct optional parameters" do # Additional coverage for optional params assert {:ok, schema} = Schema.get_schema(:call_api) diff --git a/test/quoracle/actions/schema_api_integration_test.exs b/test/quoracle/actions/schema_api_integration_test.exs deleted file mode 100644 index 0d4c1a9..0000000 --- a/test/quoracle/actions/schema_api_integration_test.exs +++ /dev/null @@ -1,304 +0,0 @@ -defmodule Quoracle.Actions.SchemaApiIntegrationTest do - @moduledoc """ - Integration tests for Schema system support of call_api action. - Tests schema retrieval, validation, and parameter checking. - """ - - use ExUnit.Case, async: true - - alias Quoracle.Actions.Schema - alias Quoracle.Actions.Validator - - describe "Schema.get_schema for call_api [UNIT]" do - test "returns schema for call_api action" do - assert {:ok, schema} = Schema.get_schema(:call_api) - - # Check required params - assert :api_type in schema.required_params - assert :url in schema.required_params - - # Check optional params - assert :method in schema.optional_params - assert :query_params in schema.optional_params - assert :body in schema.optional_params - assert :headers in schema.optional_params - assert :auth in schema.optional_params - - # GraphQL-specific - assert :query in schema.optional_params - assert :variables in schema.optional_params - - # JSON-RPC-specific - assert :rpc_method in schema.optional_params - assert :rpc_params in schema.optional_params - - # Check param types - assert schema.param_types.api_type == {:enum, [:rest, :graphql, :jsonrpc]} - assert schema.param_types.url == :string - assert schema.param_types.method == :string - assert schema.param_types.headers == :map - assert schema.param_types.body == :any - assert schema.param_types.auth == :map - end - - test "includes param descriptions for call_api" do - {:ok, schema} = Schema.get_schema(:call_api) - - assert is_binary(schema.param_descriptions.api_type) - assert String.contains?(schema.param_descriptions.api_type, "rest") - assert String.contains?(schema.param_descriptions.api_type, "graphql") - assert String.contains?(schema.param_descriptions.api_type, "jsonrpc") - - assert is_binary(schema.param_descriptions.url) - assert is_binary(schema.param_descriptions.method) - assert is_binary(schema.param_descriptions.query) - assert is_binary(schema.param_descriptions.rpc_method) - end - - test "includes consensus rules for call_api" do - {:ok, schema} = Schema.get_schema(:call_api) - - assert schema.consensus_rules.api_type == :exact_match - assert schema.consensus_rules.url == :exact_match - assert schema.consensus_rules.method == :exact_match - assert schema.consensus_rules.timeout == {:percentile, 100} - assert schema.consensus_rules.auth == :exact_match - end - end - - describe "Schema.list_actions includes call_api [UNIT]" do - test "call_api is in the list of available actions" do - actions = Schema.list_actions() - assert :call_api in actions - end - end - - describe "Schema.validate_action_type for call_api [UNIT]" do - test "validates call_api as a valid action type" do - assert {:ok, :call_api} = Schema.validate_action_type(:call_api) - end - - test "accepts call_api as atom" do - assert {:ok, :call_api} = Schema.validate_action_type(:call_api) - end - end - - describe "Schema.get_action_description for call_api [UNIT]" do - test "returns description for call_api" do - description = Schema.get_action_description(:call_api) - assert is_binary(description) - assert description =~ ~r/api/i - end - end - - describe "Schema.get_action_priority for call_api [UNIT]" do - test "returns priority for call_api" do - priority = Schema.get_action_priority(:call_api) - assert is_integer(priority) - assert priority >= 1 - assert priority <= 22 - end - end - - describe "Validator validates call_api parameters [INTEGRATION]" do - test "validates REST call_api with all required params" do - params = %{ - api_type: "rest", - url: "https://api.example.com/data", - method: "GET" - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.api_type == :rest - assert validated.url == "https://api.example.com/data" - assert validated.method == "GET" - end - - test "validates GraphQL call_api with query" do - params = %{ - api_type: "graphql", - url: "https://api.github.com/graphql", - query: "{ viewer { login } }", - variables: %{first: 10} - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.api_type == :graphql - assert validated.query == "{ viewer { login } }" - assert validated.variables == %{first: 10} - end - - test "validates JSON-RPC call_api with method and params" do - params = %{ - api_type: "jsonrpc", - url: "https://mainnet.infura.io/v3/key", - rpc_method: "eth_getBalance", - rpc_params: ["0x0000000000000000000000000000000000000000", "latest"] - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.api_type == :jsonrpc - assert validated.rpc_method == "eth_getBalance" - assert validated.rpc_params == ["0x0000000000000000000000000000000000000000", "latest"] - end - - test "rejects call_api with missing required params" do - # Missing url - params = %{ - api_type: "rest", - method: "GET" - } - - assert {:error, errors} = Validator.validate_params(:call_api, params) - assert errors == :missing_required_param - end - - test "rejects call_api with invalid api_type" do - params = %{ - api_type: "invalid", - url: "https://api.example.com" - } - - assert {:error, errors} = Validator.validate_params(:call_api, params) - assert errors == :invalid_enum_value - end - - test "rejects call_api with invalid method for REST" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "INVALID" - } - - assert {:error, errors} = Validator.validate_params(:call_api, params) - - assert errors == :invalid_http_method - end - - test "validates authentication parameters" do - # Bearer auth - bearer_params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "GET", - auth: %{ - type: "bearer", - token: "test-token" - } - } - - assert {:ok, validated} = Validator.validate_params(:call_api, bearer_params) - assert validated.auth.type == "bearer" - assert validated.auth.token == "test-token" - - # Basic auth - basic_params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "GET", - auth: %{ - type: "basic", - username: "user", - password: "pass" - } - } - - assert {:ok, validated} = Validator.validate_params(:call_api, basic_params) - assert validated.auth.type == "basic" - assert validated.auth.username == "user" - end - - test "validates headers as map" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "POST", - headers: %{ - "Content-Type" => "application/json", - "X-API-Key" => "key123" - } - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.headers["Content-Type"] == "application/json" - assert validated.headers["X-API-Key"] == "key123" - end - - test "rejects headers that are not a map" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "GET", - headers: "invalid" - } - - assert {:error, errors} = Validator.validate_params(:call_api, params) - assert errors == :invalid_param_type - end - - test "validates timeout as number" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "GET", - timeout: 30 - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.timeout == 30 - end - end - - describe "Protocol-specific validation [INTEGRATION]" do - test "REST requires method parameter" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - body: %{"data" => "test"} - } - - # Method is required for REST - assert {:error, :missing_required_param} = Validator.validate_params(:call_api, params) - end - - test "GraphQL requires query parameter" do - params = %{ - api_type: "graphql", - url: "https://api.github.com/graphql" - # Missing query - } - - assert {:error, :missing_required_param} = Validator.validate_params(:call_api, params) - end - - test "JSON-RPC requires rpc_method" do - params = %{ - api_type: "jsonrpc", - url: "https://mainnet.infura.io/v3/key" - # Missing rpc_method - } - - assert {:error, :missing_required_param} = Validator.validate_params(:call_api, params) - end - - test "validates complex nested auth structures" do - params = %{ - api_type: "rest", - url: "https://api.example.com", - method: "GET", - auth: %{ - type: "oauth2", - client_id: "client123", - client_secret: "secret456", - token_url: "https://oauth.example.com/token", - scope: "read write" - } - } - - assert {:ok, validated} = Validator.validate_params(:call_api, params) - assert validated.auth.type == "oauth2" - assert validated.auth.client_id == "client123" - assert validated.auth.token_url == "https://oauth.example.com/token" - end - end -end diff --git a/test/quoracle/actions/schema_batch_async_test.exs b/test/quoracle/actions/schema_batch_async_test.exs index 5f59f49..69cb55f 100644 --- a/test/quoracle/actions/schema_batch_async_test.exs +++ b/test/quoracle/actions/schema_batch_async_test.exs @@ -12,24 +12,6 @@ defmodule Quoracle.Actions.SchemaBatchAsyncTest do # ARC Verification Criteria from ACTION_Schema v30.0 describe "batch_async action definition (v30.0)" do - # R12: Action Registered - test "batch_async in action list" do - # [UNIT] - WHEN list_actions/0 called THEN includes :batch_async - actions = Schema.list_actions() - assert :batch_async in actions - end - - # R13: Schema Defined - test "batch_async schema exists" do - # [UNIT] - WHEN get_schema(:batch_async) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:batch_async) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R14: Required Params test "batch_async requires actions param" do # [UNIT] - WHEN get_schema(:batch_async) called THEN required_params includes :actions @@ -94,23 +76,9 @@ defmodule Quoracle.Actions.SchemaBatchAsyncTest do # [UNIT] - WHEN async_batchable?(:wait) called THEN returns false refute ActionList.async_batchable?(:wait) end - - # R24: Total Actions Updated - test "action list has 22 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 22 actions (21 + batch_async) - actions = Schema.list_actions() - assert length(actions) == 22 - end end describe "batch_async param descriptions" do - test "batch_async has param description for actions" do - # [UNIT] - WHEN get_schema(:batch_async) called THEN param_descriptions includes :actions - {:ok, schema} = Schema.get_schema(:batch_async) - assert Map.has_key?(schema.param_descriptions, :actions) - assert is_binary(schema.param_descriptions[:actions]) - end - test "batch_async actions description mentions minimum 2 actions" do # Verify LLM guidance mentions minimum requirement {:ok, schema} = Schema.get_schema(:batch_async) diff --git a/test/quoracle/actions/schema_batch_sync_test.exs b/test/quoracle/actions/schema_batch_sync_test.exs index 660a436..4357082 100644 --- a/test/quoracle/actions/schema_batch_sync_test.exs +++ b/test/quoracle/actions/schema_batch_sync_test.exs @@ -12,24 +12,6 @@ defmodule Quoracle.Actions.SchemaBatchSyncTest do # ARC Verification Criteria from ACTION_Schema v29.0 describe "batch_sync action definition (v29.0)" do - # R1: Action Registered - test "batch_sync in action list" do - # [UNIT] - WHEN list_actions/0 called THEN includes :batch_sync - actions = Schema.list_actions() - assert :batch_sync in actions - end - - # R2: Schema Defined - test "batch_sync schema exists" do - # [UNIT] - WHEN get_schema(:batch_sync) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:batch_sync) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R3: Required Params test "batch_sync requires actions param" do # [UNIT] - WHEN get_schema(:batch_sync) called THEN required_params includes :actions @@ -100,23 +82,9 @@ defmodule Quoracle.Actions.SchemaBatchSyncTest do actions = ActionList.batchable_actions() refute :batch_sync in actions end - - # R11: Total Actions Updated - test "action list has 21 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 21 actions (20 + batch_sync) - actions = Schema.list_actions() - assert length(actions) == 22 - end end describe "batch_sync param description" do - test "batch_sync has param description for actions" do - # [UNIT] - WHEN get_schema(:batch_sync) called THEN param_descriptions includes :actions - {:ok, schema} = Schema.get_schema(:batch_sync) - assert Map.has_key?(schema.param_descriptions, :actions) - assert is_binary(schema.param_descriptions[:actions]) - end - test "batch_sync actions description mentions minimum 2 actions" do # Verify LLM guidance mentions minimum requirement {:ok, schema} = Schema.get_schema(:batch_sync) diff --git a/test/quoracle/actions/schema_file_actions_test.exs b/test/quoracle/actions/schema_file_actions_test.exs index 0035f35..31ad220 100644 --- a/test/quoracle/actions/schema_file_actions_test.exs +++ b/test/quoracle/actions/schema_file_actions_test.exs @@ -15,24 +15,6 @@ defmodule Quoracle.Actions.SchemaFileActionsTest do # ARC Verification Criteria R1-R5 # =========================================================================== describe "file_read action definition (v27.0)" do - # R1: Action Registered - test "file_read in action list" do - # [UNIT] - WHEN list_actions/0 called THEN includes :file_read - actions = Schema.list_actions() - assert :file_read in actions - end - - # R2: Schema Defined - test "file_read schema exists" do - # [UNIT] - WHEN get_schema(:file_read) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:file_read) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R3: Required Params test "file_read requires path" do # [UNIT] - WHEN get_schema(:file_read) called THEN required_params includes :path @@ -82,24 +64,6 @@ defmodule Quoracle.Actions.SchemaFileActionsTest do # ARC Verification Criteria R6-R11 # =========================================================================== describe "file_write action definition (v27.0)" do - # R6: Action Registered - test "file_write in action list" do - # [UNIT] - WHEN list_actions/0 called THEN includes :file_write - actions = Schema.list_actions() - assert :file_write in actions - end - - # R7: Schema Defined - test "file_write schema exists" do - # [UNIT] - WHEN get_schema(:file_write) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:file_write) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R8: Required Params test "file_write requires path and mode" do # [UNIT] - WHEN get_schema(:file_write) called THEN required_params includes :path and :mode @@ -155,18 +119,6 @@ defmodule Quoracle.Actions.SchemaFileActionsTest do assert schema.param_types[:new_string] == :string assert schema.param_types[:replace_all] == :boolean end - - # Additional: Param Descriptions - test "file_write has param descriptions" do - # [UNIT] - WHEN get_schema(:file_write) called THEN param_descriptions exist for all params - {:ok, schema} = Schema.get_schema(:file_write) - assert Map.has_key?(schema.param_descriptions, :path) - assert Map.has_key?(schema.param_descriptions, :mode) - assert Map.has_key?(schema.param_descriptions, :content) - assert Map.has_key?(schema.param_descriptions, :old_string) - assert Map.has_key?(schema.param_descriptions, :new_string) - assert Map.has_key?(schema.param_descriptions, :replace_all) - end end # =========================================================================== @@ -197,14 +149,6 @@ defmodule Quoracle.Actions.SchemaFileActionsTest do assert Schema.get_action_priority(:file_write) == 19 end - # R14: Total Actions Updated - test "action list has 21 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 21 actions - # Previous: 19 actions + file_read + file_write = 21 - actions = Schema.list_actions() - assert length(actions) == 22 - end - # Additional: wait_required? for file actions test "file actions require wait" do # [UNIT] - file actions require subsequent wait like other actions diff --git a/test/quoracle/actions/schema_profile_test.exs b/test/quoracle/actions/schema_profile_test.exs index 96c2ffd..f767973 100644 --- a/test/quoracle/actions/schema_profile_test.exs +++ b/test/quoracle/actions/schema_profile_test.exs @@ -16,35 +16,6 @@ defmodule Quoracle.Actions.SchemaProfileTest do alias Quoracle.Actions.Validator describe "spawn_child profile parameter" do - # R7: Profile in spawn_child Schema - test "spawn_child schema includes profile parameter" do - {:ok, schema} = Schema.get_schema(:spawn_child) - - # Profile must be in required_params (per R8: profile is required) - assert :profile in schema.required_params - end - - # R8: Profile Required - test "spawn_child profile parameter is required" do - {:ok, schema} = Schema.get_schema(:spawn_child) - - # Check various ways it might be marked required - required = - cond do - Map.has_key?(schema, :required_params) -> - :profile in schema.required_params - - Map.has_key?(schema, :parameters) -> - param = Map.get(schema.parameters, :profile, %{}) - Map.get(param, :required, false) - - true -> - false - end - - assert required, "profile parameter should be required" - end - # R9: Profile Type String test "spawn_child profile has string type" do {:ok, schema} = Schema.get_schema(:spawn_child) @@ -106,21 +77,6 @@ defmodule Quoracle.Actions.SchemaProfileTest do assert {:error, :missing_required_param} = result end - test "validator accepts spawn_child with profile" do - params = %{ - "task_description" => "Test task", - "success_criteria" => "Complete", - "immediate_context" => "Test context", - "approach_guidance" => "Standard approach", - "profile" => "some-profile" - } - - result = Validator.validate_params(:spawn_child, params) - - # Should pass validation (profile existence is checked at execution time) - assert {:ok, _validated} = result - end - test "validator validates profile is string" do params = %{ "task_description" => "Test task", diff --git a/test/quoracle/actions/schema_record_cost_test.exs b/test/quoracle/actions/schema_record_cost_test.exs index 1aa4b87..bcb7d45 100644 --- a/test/quoracle/actions/schema_record_cost_test.exs +++ b/test/quoracle/actions/schema_record_cost_test.exs @@ -16,15 +16,6 @@ defmodule Quoracle.Actions.SchemaRecordCostTest do assert :record_cost in actions end - # R2: Schema Defined [UNIT] - test "R2: record_cost schema exists" do - result = Schema.get_schema(:record_cost) - assert {:ok, schema} = result - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - end - # R3: Required Params [UNIT] test "R3: record_cost requires amount only" do {:ok, schema} = Schema.get_schema(:record_cost) @@ -59,12 +50,6 @@ defmodule Quoracle.Actions.SchemaRecordCostTest do priority = Schema.get_action_priority(:record_cost) assert priority == 15 end - - # R8: Total Actions Updated [UNIT] - test "R8: action list has 21 actions" do - actions = Schema.list_actions() - assert length(actions) == 22 - end end describe "record_cost param_descriptions" do @@ -73,21 +58,6 @@ defmodule Quoracle.Actions.SchemaRecordCostTest do assert Map.has_key?(schema.param_descriptions, :amount) assert schema.param_descriptions[:amount] =~ "USD" end - - test "has description for description parameter" do - {:ok, schema} = Schema.get_schema(:record_cost) - assert Map.has_key?(schema.param_descriptions, :description) - end - - test "has description for category parameter" do - {:ok, schema} = Schema.get_schema(:record_cost) - assert Map.has_key?(schema.param_descriptions, :category) - end - - test "has description for metadata parameter" do - {:ok, schema} = Schema.get_schema(:record_cost) - assert Map.has_key?(schema.param_descriptions, :metadata) - end end describe "record_cost consensus_rules" do diff --git a/test/quoracle/actions/schema_skills_test.exs b/test/quoracle/actions/schema_skills_test.exs index 47f73b1..932cb7d 100644 --- a/test/quoracle/actions/schema_skills_test.exs +++ b/test/quoracle/actions/schema_skills_test.exs @@ -19,19 +19,6 @@ defmodule Quoracle.Actions.SchemaSkillsTest do # ========================================================================== describe "learn_skills schema (R5-R9)" do - # R5: Action Registered - test "learn_skills in action list" do - actions = Schema.list_actions() - assert :learn_skills in actions - end - - # R6: Schema Defined - test "learn_skills schema exists" do - result = Schema.get_schema(:learn_skills) - assert {:ok, schema} = result - assert is_map(schema) - end - # R7: Required Params test "learn_skills requires skills" do {:ok, schema} = Schema.get_schema(:learn_skills) @@ -57,19 +44,6 @@ defmodule Quoracle.Actions.SchemaSkillsTest do # ========================================================================== describe "create_skill schema (R10-R14)" do - # R10: Action Registered - test "create_skill in action list" do - actions = Schema.list_actions() - assert :create_skill in actions - end - - # R11: Schema Defined - test "create_skill schema exists" do - result = Schema.get_schema(:create_skill) - assert {:ok, schema} = result - assert is_map(schema) - end - # R12: Required Params test "create_skill requires name, description, content" do {:ok, schema} = Schema.get_schema(:create_skill) @@ -109,20 +83,5 @@ defmodule Quoracle.Actions.SchemaSkillsTest do assert String.contains?(desc, "WHEN"), "#{action} description should have WHEN guidance" end end - - # R16: Priorities Defined - test "skill actions have priorities" do - for action <- [:learn_skills, :create_skill] do - priority = Schema.get_action_priority(action) - assert is_integer(priority), "#{action} should have integer priority" - assert priority > 0, "#{action} priority should be positive" - end - end - - # R17: Total Actions Updated - test "action list has 20 actions" do - actions = Schema.list_actions() - assert length(actions) == 22 - end end end diff --git a/test/quoracle/actions/schema_test.exs b/test/quoracle/actions/schema_test.exs index 9a5b503..295b8a2 100644 --- a/test/quoracle/actions/schema_test.exs +++ b/test/quoracle/actions/schema_test.exs @@ -44,7 +44,7 @@ defmodule Quoracle.Actions.SchemaTest do end describe "list_actions/0" do - test "returns all 18 action atoms" do + test "returns all action atoms" do actions = Schema.list_actions() assert length(actions) == 22 end @@ -63,10 +63,6 @@ defmodule Quoracle.Actions.SchemaTest do assert Schema.get_action_priority(:send_message) == 3 end - test "returns priority 6 for fetch_web" do - assert Schema.get_action_priority(:fetch_web) == 6 - end - test "returns priority 10 for answer_engine" do assert Schema.get_action_priority(:answer_engine) == 10 end @@ -79,10 +75,6 @@ defmodule Quoracle.Actions.SchemaTest do assert Schema.get_action_priority(:call_mcp) == 16 end - test "returns priority 17 for call_api" do - assert Schema.get_action_priority(:call_api) == 17 - end - test "returns priority 18 for execute_shell" do assert Schema.get_action_priority(:execute_shell) == 18 end @@ -135,16 +127,6 @@ defmodule Quoracle.Actions.SchemaTest do assert Enum.all?(values, &is_integer/1) assert Enum.all?(values, &(&1 > 0)) end - - test "orient has lowest priority (1)" do - priorities = Schema.get_priorities() - assert priorities.orient == 1 - end - - test "spawn_child has highest priority (22)" do - priorities = Schema.get_priorities() - assert priorities.spawn_child == 22 - end end describe "validate_action_type/1" do @@ -175,18 +157,6 @@ defmodule Quoracle.Actions.SchemaTest do end end - test "returns true for :spawn_child" do - assert Schema.wait_required?(:spawn_child) == true - end - - test "returns true for :orient" do - assert Schema.wait_required?(:orient) == true - end - - test "returns true for :send_message" do - assert Schema.wait_required?(:send_message) == true - end - test "raises for unknown action" do assert_raise FunctionClauseError, fn -> Schema.wait_required?(:invalid_action) @@ -352,97 +322,6 @@ defmodule Quoracle.Actions.SchemaTest do end end - describe "fetch_web action definition" do - test "has url as required param" do - {:ok, schema} = Schema.get_schema(:fetch_web) - assert :url in schema.required_params - end - - test "has correct optional params" do - {:ok, schema} = Schema.get_schema(:fetch_web) - assert :security_check in schema.optional_params - assert :timeout in schema.optional_params - assert :user_agent in schema.optional_params - assert :follow_redirects in schema.optional_params - end - - test "has correct param types" do - {:ok, schema} = Schema.get_schema(:fetch_web) - assert schema.param_types.url == :string - assert schema.param_types.security_check == :boolean - assert schema.param_types.timeout == :number - assert schema.param_types.user_agent == :string - assert schema.param_types.follow_redirects == :boolean - end - end - - describe "call_api action definition" do - test "has correct required params" do - {:ok, schema} = Schema.get_schema(:call_api) - assert :api_type in schema.required_params - assert :url in schema.required_params - end - - test "has correct optional params including auth" do - {:ok, schema} = Schema.get_schema(:call_api) - assert :method in schema.optional_params - assert :headers in schema.optional_params - assert :auth in schema.optional_params - assert :query in schema.optional_params - end - - test "has correct consensus rules" do - {:ok, schema} = Schema.get_schema(:call_api) - assert schema.consensus_rules.api_type == :exact_match - assert schema.consensus_rules.url == :exact_match - assert schema.consensus_rules.method == :exact_match - assert schema.consensus_rules.timeout == {:percentile, 100} - assert schema.consensus_rules.max_body_size == {:percentile, 100} - assert schema.consensus_rules.auth == :exact_match - end - end - - describe "call_mcp action definition" do - test "has correct optional params for agent-driven discovery" do - {:ok, schema} = Schema.get_schema(:call_mcp) - - # v20.0: All params optional (XOR handles mutual exclusion) - expected_optional = [ - :transport, - :command, - :url, - :connection_id, - :tool, - :arguments, - :terminate, - :timeout - ] - - Enum.each(expected_optional, fn param -> - assert param in schema.optional_params, - "Expected #{inspect(param)} in optional_params" - end) - end - - test "has exact match consensus rules for connection params" do - {:ok, schema} = Schema.get_schema(:call_mcp) - # v20.0: Connection-related params use exact_match - assert schema.consensus_rules.transport == :exact_match - assert schema.consensus_rules.command == :exact_match - assert schema.consensus_rules.url == :exact_match - assert schema.consensus_rules.connection_id == :exact_match - assert schema.consensus_rules.tool == :exact_match - assert schema.consensus_rules.arguments == :exact_match - assert schema.consensus_rules.terminate == :exact_match - end - - test "has percentile consensus rule for timeout" do - {:ok, schema} = Schema.get_schema(:call_mcp) - # v20.0: Timeout uses percentile 50 for numeric consensus - assert schema.consensus_rules.timeout == {:percentile, 50} - end - end - # ACTION_Schema v20.0 - MCP Action Schema Update # ARC Verification Criteria for call_mcp agent-driven discovery redesign describe "call_mcp v20.0 schema (agent-driven discovery)" do @@ -513,24 +392,6 @@ defmodule Quoracle.Actions.SchemaTest do end end - describe "performance" do - test "get_schema/1 completes in O(1) time" do - # Warm up - Schema.get_schema(:spawn_child) - - # Measure time for multiple lookups - times = - for _ <- 1..100 do - {time, _} = :timer.tc(fn -> Schema.get_schema(:spawn_child) end) - time - end - - avg_time = Enum.sum(times) / length(times) - # Should be very fast, under 100 microseconds - assert avg_time < 100 - end - end - # ACTION_Schema v22.0 - Dismiss Child Action # ARC Verification Criteria for dismiss_child action registration # WorkGroupID: feat-20251224-dismiss-child @@ -543,17 +404,6 @@ defmodule Quoracle.Actions.SchemaTest do assert :dismiss_child in actions end - # R2: Schema Defined - test "dismiss_child schema exists" do - # [UNIT] - WHEN get_schema(:dismiss_child) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:dismiss_child) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R3: Required Params test "dismiss_child requires child_id" do # [UNIT] - WHEN get_schema(:dismiss_child) called THEN required_params includes :child_id @@ -598,13 +448,6 @@ defmodule Quoracle.Actions.SchemaTest do # [UNIT] - WHEN get_action_priority(:dismiss_child) called THEN returns 20 assert Schema.get_action_priority(:dismiss_child) == 20 end - - # R9: Total Actions Updated - test "action list has 21 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 21 actions (includes record_cost) - actions = Schema.list_actions() - assert length(actions) == 22 - end end # ACTION_Schema v21.0 - Search Secrets Action @@ -617,17 +460,6 @@ defmodule Quoracle.Actions.SchemaTest do assert :search_secrets in actions end - # R2: Schema Defined - test "search_secrets schema exists" do - # [UNIT] - WHEN get_schema(:search_secrets) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:search_secrets) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R3: Required Params test "search_secrets requires search_terms" do # [UNIT] - WHEN get_schema(:search_secrets) called THEN required_params includes :search_terms @@ -671,13 +503,6 @@ defmodule Quoracle.Actions.SchemaTest do # [UNIT] - WHEN get_action_priority(:search_secrets) called THEN returns 8 assert Schema.get_action_priority(:search_secrets) == 8 end - - # R9: Total Actions Updated - test "action list has 21 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 21 actions (includes record_cost) - actions = Schema.list_actions() - assert length(actions) == 22 - end end # ACTION_Schema v22.0 - Announcement Target for SendMessage @@ -737,17 +562,6 @@ defmodule Quoracle.Actions.SchemaTest do assert :generate_images in actions end - # R2: Schema Defined - test "generate_images schema exists" do - # [UNIT] - WHEN get_schema(:generate_images) called THEN returns valid schema - assert {:ok, schema} = Schema.get_schema(:generate_images) - assert is_map(schema) - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - end - # R3: Required Params test "generate_images requires prompt" do # [UNIT] - WHEN get_schema(:generate_images) called THEN required_params includes :prompt @@ -793,13 +607,6 @@ defmodule Quoracle.Actions.SchemaTest do assert Schema.get_action_priority(:generate_images) == 14 end - # R9: Total Actions Updated - test "action list has 21 actions" do - # [UNIT] - WHEN list_actions/0 called THEN returns 21 actions (includes record_cost) - actions = Schema.list_actions() - assert length(actions) == 22 - end - # Additional: Param Descriptions test "generate_images has param descriptions" do # [UNIT] - WHEN get_schema(:generate_images) called THEN param_descriptions exist diff --git a/test/quoracle/actions/schema_todo_enhancement_test.exs b/test/quoracle/actions/schema_todo_enhancement_test.exs deleted file mode 100644 index ee385a5..0000000 --- a/test/quoracle/actions/schema_todo_enhancement_test.exs +++ /dev/null @@ -1,264 +0,0 @@ -defmodule Quoracle.Actions.SchemaTodoEnhancementTest do - @moduledoc """ - Tests for the enhanced TODO action schema with explicit nested structure. - This ensures LLMs receive proper field definitions for TODO items. - - Tests verify the TODO schema includes: - - Nested map structure with explicit properties - - Enum type for state field - - Proper consensus rules - """ - use ExUnit.Case, async: true - alias Quoracle.Actions.Schema - - describe "todo action enhanced schema" do - test "returns enhanced nested structure for todo action" do - assert {:ok, schema} = Schema.get_schema(:todo) - - # Verify basic structure - assert schema.required_params == [:items] - assert schema.optional_params == [] - - # CRITICAL: Verify the enhanced nested type structure - assert schema.param_types.items == - {:list, - {:map, - %{ - content: :string, - state: {:enum, [:todo, :pending, :done]} - }}} - end - - test "todo items param_type has nested map with content and state fields" do - {:ok, schema} = Schema.get_schema(:todo) - - # Extract the nested type structure - assert {:list, inner_type} = schema.param_types.items - assert {:map, properties} = inner_type - - # Verify the exact property definitions - assert properties.content == :string - assert properties.state == {:enum, [:todo, :pending, :done]} - end - - test "todo state field uses enum type with specific values" do - {:ok, schema} = Schema.get_schema(:todo) - - # Navigate to the state enum definition - {:list, {:map, properties}} = schema.param_types.items - {:enum, allowed_values} = properties.state - - # Verify exact enum values - assert allowed_values == [:todo, :pending, :done] - assert length(allowed_values) == 3 - end - - test "todo schema defines exactly two properties: content and state" do - {:ok, schema} = Schema.get_schema(:todo) - - # Extract property map - {:list, {:map, properties}} = schema.param_types.items - - # Verify only content and state exist - assert Map.keys(properties) |> Enum.sort() == [:content, :state] - assert map_size(properties) == 2 - end - - test "todo consensus rule uses semantic_similarity with 0.85 threshold" do - {:ok, schema} = Schema.get_schema(:todo) - - # Verify consensus rule remains unchanged - assert schema.consensus_rules.items == {:semantic_similarity, threshold: 0.85} - end - - test "todo action has priority 11 in action priorities" do - # This ensures TODO action is properly integrated - assert Schema.get_action_priority(:todo) == 11 - end - - test "todo is included in list_actions" do - actions = Schema.list_actions() - assert :todo in actions - end - - test "validate_action_type accepts todo" do - assert {:ok, :todo} = Schema.validate_action_type(:todo) - end - - test "wait_required? returns true for todo action" do - assert Schema.wait_required?(:todo) == true - end - end - - describe "nested type structure validation" do - test "nested map type is not the same as generic map" do - {:ok, todo_schema} = Schema.get_schema(:todo) - {:ok, api_schema} = Schema.get_schema(:call_api) - - # TODO uses nested map with properties - assert {:list, {:map, %{}}} = todo_schema.param_types.items - - # call_api.headers uses generic map (no nested structure) - assert api_schema.param_types.headers == :map - - # They should be different - refute todo_schema.param_types.items == {:list, :map} - end - - test "enum type provides constrained values" do - {:ok, schema} = Schema.get_schema(:todo) - {:list, {:map, properties}} = schema.param_types.items - - # State uses enum type, not plain atom - assert {:enum, _values} = properties.state - refute properties.state == :atom - refute properties.state == :string - end - end - - describe "backward compatibility" do - test "other actions still return their original schemas" do - # Verify wait action has new union type (breaking change) - {:ok, wait_schema} = Schema.get_schema(:wait) - assert wait_schema.optional_params == [:wait] - assert wait_schema.param_types.wait == {:union, [:boolean, :number]} - - # Verify orient action unchanged - {:ok, orient_schema} = Schema.get_schema(:orient) - assert :current_situation in orient_schema.required_params - assert orient_schema.param_types.current_situation == :string - - # Verify spawn_child has profile as required (v24.0) - {:ok, spawn_schema} = Schema.get_schema(:spawn_child) - - assert spawn_schema.required_params == [ - :task_description, - :success_criteria, - :immediate_context, - :approach_guidance, - :profile - ] - - assert spawn_schema.param_types.task_description == :string - end - - test "all 21 actions remain accessible" do - actions = Schema.list_actions() - assert length(actions) == 22 - assert :record_cost in actions - end - end - - describe "type system completeness" do - test "schema supports all required type representations" do - {:ok, todo_schema} = Schema.get_schema(:todo) - - # Verify the type system can represent: - # 1. Lists - assert {:list, _} = todo_schema.param_types.items - - # 2. Nested maps with properties - {:list, nested} = todo_schema.param_types.items - assert {:map, %{}} = nested - - # 3. Enums - {:list, {:map, props}} = todo_schema.param_types.items - assert {:enum, _} = props.state - - # 4. Basic types - assert props.content == :string - end - - test "nested structure is properly recursive" do - {:ok, schema} = Schema.get_schema(:todo) - - # Unpack the nested structure step by step - items_type = schema.param_types.items - assert {:list, inner} = items_type - - assert {:map, properties} = inner - assert is_map(properties) - - assert Map.has_key?(properties, :content) - assert Map.has_key?(properties, :state) - - # Each property has its own type - assert properties.content == :string - assert {:enum, values} = properties.state - assert is_list(values) - end - end - - describe "error handling for enhanced types" do - test "get_schema still returns error for invalid actions" do - assert {:error, :unknown_action} = Schema.get_schema(:not_todo) - assert {:error, :unknown_action} = Schema.get_schema(:invalid) - assert {:error, :unknown_action} = Schema.get_schema(nil) - end - - test "get_schema handles todo action without errors" do - assert {:ok, schema} = Schema.get_schema(:todo) - assert is_map(schema) - assert Map.has_key?(schema, :param_types) - end - end - - describe "integration with consensus rules" do - test "todo items consensus rule works with nested structure" do - {:ok, schema} = Schema.get_schema(:todo) - - # Consensus rule should apply to the entire items list - assert Map.has_key?(schema.consensus_rules, :items) - assert {:semantic_similarity, opts} = schema.consensus_rules.items - assert opts[:threshold] == 0.85 - end - - test "all todo params have consensus rules defined" do - {:ok, schema} = Schema.get_schema(:todo) - - # Every required param must have a consensus rule - for param <- schema.required_params do - assert Map.has_key?(schema.consensus_rules, param), - "Missing consensus rule for #{param}" - end - - # Every optional param must have a consensus rule (none for TODO) - for param <- schema.optional_params do - assert Map.has_key?(schema.consensus_rules, param), - "Missing consensus rule for #{param}" - end - end - end - - describe "property-based testing for type consistency" do - test "all enum values are atoms" do - {:ok, schema} = Schema.get_schema(:todo) - {:list, {:map, props}} = schema.param_types.items - {:enum, values} = props.state - - assert Enum.all?(values, &is_atom/1), - "All enum values should be atoms" - end - - test "nested map properties are all atoms" do - {:ok, schema} = Schema.get_schema(:todo) - {:list, {:map, props}} = schema.param_types.items - - assert Enum.all?(Map.keys(props), &is_atom/1), - "All property keys should be atoms" - end - - test "type definitions are properly nested without cycles" do - {:ok, schema} = Schema.get_schema(:todo) - - # Traverse the type tree to ensure no circular references - assert {:list, inner} = schema.param_types.items - assert {:map, props} = inner - # Terminal type - assert props.content == :string - assert {:enum, values} = props.state - # Terminal values - assert Enum.all?(values, &is_atom/1) - end - end -end diff --git a/test/quoracle/actions/schema_todo_test.exs b/test/quoracle/actions/schema_todo_test.exs index b8bcfe0..ce71456 100644 --- a/test/quoracle/actions/schema_todo_test.exs +++ b/test/quoracle/actions/schema_todo_test.exs @@ -7,8 +7,6 @@ defmodule Quoracle.Actions.SchemaTodoTest do test "todo action is included in @actions list" do actions = Schema.list_actions() assert :todo in actions - # Should have 21 actions now (including record_cost) - assert length(actions) == 22 end test "get_schema/1 returns todo schema" do @@ -36,51 +34,11 @@ defmodule Quoracle.Actions.SchemaTodoTest do } end - test "todo schema has correct required params" do - assert {:ok, schema} = Schema.get_schema(:todo) - assert schema.required_params == [:items] - end - - test "todo schema has no optional params" do - assert {:ok, schema} = Schema.get_schema(:todo) - assert schema.optional_params == [] - end - - test "todo schema defines items as list of maps with nested structure" do - assert {:ok, schema} = Schema.get_schema(:todo) - - assert schema.param_types.items == - {:list, - {:map, - %{ - content: :string, - state: {:enum, [:todo, :pending, :done]} - }}} - end - - test "todo schema uses semantic similarity for consensus" do - assert {:ok, schema} = Schema.get_schema(:todo) - assert schema.consensus_rules.items == {:semantic_similarity, threshold: 0.85} - end - - test "todo schema has high similarity threshold" do - assert {:ok, schema} = Schema.get_schema(:todo) - {:semantic_similarity, threshold: threshold} = schema.consensus_rules.items - assert threshold == 0.85 - end - test "wait_required?/1 returns true for todo action" do # Only :wait action returns false, all others including :todo return true assert Schema.wait_required?(:todo) end - test "get_action_priority/1 returns priority for todo" do - # Todo should have a defined priority for consensus tiebreaking - priority = Schema.get_action_priority(:todo) - assert is_integer(priority) - assert priority >= 0 - end - test "todo action priority is between safer and riskier actions" do # Todo (6) should be between orient (1) and spawn_child (11) todo_priority = Schema.get_action_priority(:todo) @@ -93,54 +51,5 @@ defmodule Quoracle.Actions.SchemaTodoTest do # todo (6) < spawn_child (11) - todo is less consequential than spawn_child assert todo_priority < spawn_priority end - - test "todo schema is compatible with validator" do - # The schema should be structured correctly for ACTION_Validator - assert {:ok, schema} = Schema.get_schema(:todo) - - assert Map.has_key?(schema, :required_params) - assert Map.has_key?(schema, :optional_params) - assert Map.has_key?(schema, :param_types) - assert Map.has_key?(schema, :consensus_rules) - - assert is_list(schema.required_params) - assert is_list(schema.optional_params) - assert is_map(schema.param_types) - assert is_map(schema.consensus_rules) - end - end - - describe "integration with other actions" do - test "todo schema structure matches other action schemas" do - assert {:ok, todo_schema} = Schema.get_schema(:todo) - assert {:ok, wait_schema} = Schema.get_schema(:wait) - - # Should have same top-level keys (all schemas now have param_descriptions) - assert MapSet.new(Map.keys(todo_schema)) == MapSet.new(Map.keys(wait_schema)) - end - - test "todo doesn't interfere with existing actions" do - # Existing actions should still work (using correct action names) - assert {:ok, _} = Schema.get_schema(:wait) - assert {:ok, _} = Schema.get_schema(:spawn_child) - assert {:ok, _} = Schema.get_schema(:send_message) - assert {:ok, _} = Schema.get_schema(:orient) - assert {:ok, _} = Schema.get_schema(:answer_engine) - assert {:ok, _} = Schema.get_schema(:fetch_web) - assert {:ok, _} = Schema.get_schema(:execute_shell) - assert {:ok, _} = Schema.get_schema(:call_api) - assert {:ok, _} = Schema.get_schema(:call_mcp) - end - - test "all 21 actions have schemas defined" do - actions = Schema.list_actions() - assert length(actions) == 22 - - for action <- actions do - assert {:ok, schema} = Schema.get_schema(action) - assert is_map(schema), "Missing schema for action: #{action}" - assert Map.has_key?(schema, :required_params) - end - end end end diff --git a/test/quoracle/actions/schema_web_test.exs b/test/quoracle/actions/schema_web_test.exs index 1c6f7db..2c0d80e 100644 --- a/test/quoracle/actions/schema_web_test.exs +++ b/test/quoracle/actions/schema_web_test.exs @@ -183,10 +183,6 @@ defmodule Quoracle.Actions.SchemaWebTest do end describe "fetch_web priority" do - test "maintains priority 6 (read-only external)" do - assert Schema.get_action_priority(:fetch_web) == 6 - end - test "is less risky than call_api" do fetch_priority = Schema.get_action_priority(:fetch_web) api_priority = Schema.get_action_priority(:call_api) diff --git a/test/quoracle/actions/shell_notification_fix_test.exs b/test/quoracle/actions/shell_notification_fix_test.exs index 9d9535f..191725f 100644 --- a/test/quoracle/actions/shell_notification_fix_test.exs +++ b/test/quoracle/actions/shell_notification_fix_test.exs @@ -381,31 +381,6 @@ defmodule Quoracle.Actions.ShellNotificationFixTest do end describe "Updated test assertions from old protocol" do - @tag :integration - test "async command notifies via action_result instead of shell_completed", %{ - opts: opts, - action_id: action_id - } do - # Force async mode with smart_threshold: 0 (don't rely on timing) - opts_async = Keyword.put(opts, :smart_threshold, 0) - - # OLD assertion (should fail): - # assert_receive {:shell_completed, cmd_id, result}, 30_000 - # NEW assertion: - {:ok, %{command_id: _cmd_id}} = - Shell.execute( - %{command: "sleep 0.2 && echo async"}, - "agent-001", - opts_async - ) - - assert_receive {:"$gen_cast", {:action_result, ^action_id, {:ok, result}, _opts}}, 30_000 - assert result.stdout =~ "async" - assert result.exit_code == 0 - assert result.status == :completed - assert result.sync == false - end - @tag :integration test "multiple async commands use correct action_ids", %{ pubsub: pubsub, diff --git a/test/quoracle/actions/shell_packet2_test.exs b/test/quoracle/actions/shell_packet2_test.exs index c6d9ddd..4fc72b0 100644 --- a/test/quoracle/actions/shell_packet2_test.exs +++ b/test/quoracle/actions/shell_packet2_test.exs @@ -605,6 +605,8 @@ defmodule Quoracle.Actions.ShellPacket2Test do # Isolation is inherent - each Router handles exactly one command # Helper to spawn Router and execute command + # Uses smart_threshold: :infinity to force synchronous completion, + # avoiding non-deterministic sync/async race with fast commands. spawn_and_execute = fn agent_id, command -> action_id = "action-#{agent_id}" @@ -623,17 +625,17 @@ defmodule Quoracle.Actions.ShellPacket2Test do pubsub: pubsub, router_pid: router, action_id: action_id, - smart_threshold: 0 + smart_threshold: :infinity ] - {:ok, %{command_id: cmd_id}} = Shell.execute(%{command: command}, agent_id, opts) - {action_id, cmd_id, router} + {:ok, result} = Shell.execute(%{command: command}, agent_id, opts) + {result, router} end - # Execute three commands concurrently with separate Routers - {action1, _cmd1, router1} = spawn_and_execute.("agent-1", "echo agent1") - {action2, _cmd2, router2} = spawn_and_execute.("agent-2", "echo agent2") - {action3, _cmd3, router3} = spawn_and_execute.("agent-3", "echo agent3") + # Execute three commands with separate Routers (isolation via per-action Router) + {result1, router1} = spawn_and_execute.("agent-1", "echo agent1") + {result2, router2} = spawn_and_execute.("agent-2", "echo agent2") + {result3, router3} = spawn_and_execute.("agent-3", "echo agent3") on_exit(fn -> for r <- [router1, router2, router3] do @@ -647,18 +649,7 @@ defmodule Quoracle.Actions.ShellPacket2Test do end end) - # Collect all results (order may vary) - results = - for _ <- 1..3 do - assert_receive {:"$gen_cast", {:action_result, action_id, {:ok, result}, _opts}}, 30_000 - {action_id, result} - end - # Verify each agent got its own isolated result - result1 = Enum.find_value(results, fn {id, r} -> if id == action1, do: r end) - result2 = Enum.find_value(results, fn {id, r} -> if id == action2, do: r end) - result3 = Enum.find_value(results, fn {id, r} -> if id == action3, do: r end) - assert result1.stdout =~ "agent1" assert result2.stdout =~ "agent2" assert result3.stdout =~ "agent3" diff --git a/test/quoracle/actions/shell_test.exs b/test/quoracle/actions/shell_test.exs index e605eeb..3027dad 100644 --- a/test/quoracle/actions/shell_test.exs +++ b/test/quoracle/actions/shell_test.exs @@ -56,15 +56,6 @@ defmodule Quoracle.Actions.ShellTest do assert stdout == "hello\n" end - test "captures stdout correctly for fast command", %{opts: opts} do - # Use high threshold to guarantee sync under any load - opts = Keyword.put(opts, :smart_threshold, 1000) - result = Shell.execute(%{command: "echo 'test output'"}, "agent-1", opts) - - assert {:ok, %{stdout: stdout}} = result - assert stdout == "test output\n" - end - test "captures stderr for fast failing command", %{opts: opts} do # Use high threshold to guarantee sync under any load opts = Keyword.put(opts, :smart_threshold, 1000) @@ -428,14 +419,6 @@ defmodule Quoracle.Actions.ShellTest do assert occurrences == 1, "Command executed #{occurrences} times instead of once" end - test "fast commands return sync results", %{opts: opts} do - # [BEHAVIORAL] Fast commands should complete synchronously - # Use high threshold to guarantee sync - opts = Keyword.put(opts, :smart_threshold, 1000) - result = Shell.execute(%{command: "echo 'fast'"}, "agent-1", opts) - assert {:ok, %{stdout: "fast\n"}} = result - end - test "slow commands return async results", %{opts: opts} do # [BEHAVIORAL] Slow commands should return immediately with command_id result = Shell.execute(%{command: "sleep 0.2 && echo 'slow'"}, "agent-1", opts) @@ -499,32 +482,9 @@ defmodule Quoracle.Actions.ShellTest do assert {:ok, %{exit_code: code}} = result assert code != 0 end - - test "handles working_dir that doesn't exist", %{opts: opts} do - # [UNIT] F1.5 - result = - Shell.execute( - %{command: "pwd", working_dir: "/this/path/does/not/exist"}, - "agent-1", - opts - ) - - assert {:error, :invalid_working_dir} = result - end end describe "working directory" do - test "executes in specified working_dir", %{opts: opts} do - # [UNIT] F1.3 - # Use higher threshold to avoid flaky async returns under load - opts = Keyword.put(opts, :smart_threshold, 5000) - tmp_dir = System.tmp_dir!() - result = Shell.execute(%{command: "pwd", working_dir: tmp_dir}, "agent-1", opts) - - assert {:ok, %{stdout: output}} = result - assert String.trim(output) == tmp_dir - end - test "defaults to /tmp when working_dir not provided", %{opts: opts} do # Use higher threshold to avoid flaky async returns under load opts = Keyword.put(opts, :smart_threshold, 5000) @@ -538,16 +498,5 @@ defmodule Quoracle.Actions.ShellTest do # - "resolves relative paths in working_dir" # - "handles working_dir with spaces" # These are edge cases not critical for core Shell functionality - - test "validates working_dir exists before execution", %{opts: opts} do - result = - Shell.execute( - %{command: "echo test", working_dir: "/definitely/not/a/real/path"}, - "agent-1", - opts - ) - - assert {:error, :invalid_working_dir} = result - end end end diff --git a/test/quoracle/actions/skill_actions_test.exs b/test/quoracle/actions/skill_actions_test.exs index 78d1e4d..e9a5209 100644 --- a/test/quoracle/actions/skill_actions_test.exs +++ b/test/quoracle/actions/skill_actions_test.exs @@ -8,7 +8,7 @@ defmodule Quoracle.Actions.SkillActionsTest do - ACTION_CreateSkill: R1-R12 """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Actions.LearnSkills alias Quoracle.Actions.CreateSkill @@ -444,49 +444,4 @@ defmodule Quoracle.Actions.SkillActionsTest do assert result.name == "string-keys" end end - - # =========================================================================== - # Router Integration Tests (R7, R11, R12) - # =========================================================================== - - describe "Router integration" do - @describetag :router_integration - - # Note: These tests require Router implementation that dispatches to action modules - # The Router needs to recognize :learn_skills, :create_skill actions - - test "[LearnSkills R11] Router.execute dispatches to LearnSkills", %{ - base_name: base_name, - skills_path: path - } do - create_test_skill(base_name, "router-learn-skill") - - {:ok, result} = - LearnSkills.execute( - %{skills: ["router-learn-skill"]}, - "agent-1", - skills_path: path - ) - - assert result.action == "learn_skills" - end - - test "[CreateSkill R12] Router.execute dispatches to CreateSkill", %{ - base_name: _base_name, - skills_path: path - } do - {:ok, result} = - CreateSkill.execute( - %{ - name: "router-create-skill", - description: "Via router", - content: "Content" - }, - "agent-1", - skills_path: path - ) - - assert result.action == "create_skill" - end - end end diff --git a/test/quoracle/actions/spawn_async_test.exs b/test/quoracle/actions/spawn_async_test.exs index 43ad637..f38b6bb 100644 --- a/test/quoracle/actions/spawn_async_test.exs +++ b/test/quoracle/actions/spawn_async_test.exs @@ -98,13 +98,11 @@ defmodule Quoracle.Actions.SpawnAsyncTest do end describe "R1: Spawn Returns Immediately" do - # R1: WHEN spawn_child executed THEN returns {:ok, %{agent_id: ...}} within 300ms - # (Using 300ms threshold to allow for system overhead under load) + # R1: WHEN spawn_child executed THEN returns {:ok, %{agent_id: ...}} before background work completes test "spawn_child returns immediately without blocking on LLM", %{ deps: deps, profile: profile } do - # Setup: Long narrative that would trigger LLM summarization params = %{ "task_description" => "Analyze the codebase", "success_criteria" => "Complete analysis", @@ -113,16 +111,19 @@ defmodule Quoracle.Actions.SpawnAsyncTest do "profile" => profile.name } - # Mock dynsup that simulates slow ConfigBuilder (LLM summarization) - # In production, the LLM call takes 500ms-5s. We simulate 500ms here. - # The test asserts < 300ms, so this MUST fail with sync spawn. - # (Using larger margins to avoid flaky failures under system load) + test_pid = self() + + # Mock dynsup that blocks until test explicitly unblocks it. + # If spawn were synchronous, it would deadlock here (mock waits + # for :proceed, but :proceed is only sent after spawn returns). deps_with_mock = Map.put(deps, :dynsup_fn, fn _pid, _config, _opts -> - # Simulate slow background work using receive/after (not Process.sleep) + send(test_pid, {:mock_started, self()}) + receive do + :proceed -> :ok after - 500 -> :ok + 30_000 -> :ok end pid = spawn_link(fn -> :timer.sleep(:infinity) end) @@ -132,57 +133,18 @@ defmodule Quoracle.Actions.SpawnAsyncTest do opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - # Time the spawn call - should return in < 300ms even with slow background work - {time_micros, result} = - :timer.tc(fn -> - Spawn.execute(params, "parent-1", opts) - end) - - # MUST return {:ok, _} immediately - assert {:ok, spawn_result} = result + # Spawn MUST return before mock completes (async dispatch) + {:ok, spawn_result} = Spawn.execute(params, "parent-1", opts) assert is_binary(spawn_result.agent_id) - # MUST complete in < 1000ms (1,000,000 microseconds) - # Mock sleeps 500ms in background, so async spawn returns quickly - # Using 1000ms threshold to allow for system overhead under heavy CI load - # (500ms mock delay + 500ms margin for scheduling variance) - assert time_micros < 1_000_000, - "Spawn took #{time_micros / 1000}ms, expected < 1000ms. " <> - "Spawn should return immediately, deferring child creation to background." - - # Wait for background task to complete before test cleanup - wait_for_spawn_complete(spawn_result.agent_id) - end - end - - describe "R2: Child ID Format Unchanged" do - # R2: WHEN spawn_child executed THEN child_id matches "agent-{uuid}" format - test "child_id format is agent-uuid", %{deps: deps, profile: profile} do - params = %{ - "task_description" => "Test task", - "success_criteria" => "Complete", - "immediate_context" => "Test", - "approach_guidance" => "Standard", - "profile" => profile.name - } - - deps_with_mock = - Map.put(deps, :dynsup_fn, fn _pid, _config, _opts -> - pid = spawn_link(fn -> :timer.sleep(:infinity) end) - track_pid(deps, pid) - {:ok, pid} - end) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - {:ok, result} = Spawn.execute(params, "parent-1", opts) + # Mock is still blocking — spawn returned before background work finished + assert_receive {:mock_started, mock_pid}, 5000 - # Verify UUID format: agent-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - assert result.agent_id =~ - ~r/^agent-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + # Unblock mock so background task can complete + send(mock_pid, :proceed) # Wait for background task to complete before test cleanup - wait_for_spawn_complete(result.agent_id) - # Cleanup handled by on_exit via pids_tracker + wait_for_spawn_complete(spawn_result.agent_id) end end @@ -209,6 +171,7 @@ defmodule Quoracle.Actions.SpawnAsyncTest do Map.merge(deps, %{ dynsup_fn: fn _pid, config, _opts -> agent_id = config.agent_id + mock_pid = self() # Spawn child that registers ITSELF (not the background Task) child_pid = @@ -220,19 +183,19 @@ defmodule Quoracle.Actions.SpawnAsyncTest do agent_id: agent_id }) + # Notify both mock and test that registration is complete + send(mock_pid, :child_registered_internal) send(test_pid, {:child_registered, child_registered, self()}) :timer.sleep(:infinity) end) track_pid(deps, child_pid) - # Wait for child to register before returning + # Wait for child to actually register (event-based, not time-based) receive do - {:child_registered, ^child_registered, ^child_pid} -> - # Re-send so test can also receive it - send(test_pid, {:child_registered, child_registered, child_pid}) + :child_registered_internal -> :ok after - 1000 -> :ok + 5000 -> :ok end {:ok, child_pid} @@ -263,55 +226,6 @@ defmodule Quoracle.Actions.SpawnAsyncTest do end end - describe "R4: Child Receives Initial Message" do - # R4: WHEN background spawn completes THEN child receives task message - test "child receives initial task message after background spawn", %{ - deps: deps, - profile: profile - } do - task_description = "Analyze the security vulnerabilities" - - params = %{ - "task_description" => task_description, - "success_criteria" => "Find all issues", - "immediate_context" => "Production app", - "approach_guidance" => "Focus on OWASP", - "profile" => profile.name - } - - deps_with_mock = - Map.merge(deps, %{ - dynsup_fn: fn _pid, config, _opts -> - child_pid = spawn_link(fn -> :timer.sleep(:infinity) end) - track_pid(deps, child_pid) - - Registry.register(deps.registry, {:agent, config.agent_id}, %{ - pid: child_pid, - agent_id: config.agent_id - }) - - {:ok, child_pid} - end - }) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - {:ok, result} = Spawn.execute(params, "parent-1", opts) - - # In current sync implementation, the child receives message immediately - # In async pattern, message would be sent after background spawn completes - # Either way, the broadcast should contain the task - assert_receive {:agent_spawned, broadcast}, 30_000 - assert broadcast.task == task_description - - # The child_id should match what spawn returned - assert broadcast.agent_id == result.agent_id - - # Wait for background task to complete before test cleanup - wait_for_spawn_complete(result.agent_id) - # Cleanup handled by on_exit via pids_tracker - end - end - describe "R5: Spawn Failure Notification" do # R5: WHEN background spawn fails THEN parent receives failure message via send_message test "parent receives spawn_failed message on background failure", %{ @@ -455,54 +369,6 @@ defmodule Quoracle.Actions.SpawnAsyncTest do end end - describe "R10: Concurrent Spawns" do - # R10: WHEN multiple spawn_child actions execute concurrently - # THEN all children created with unique IDs - test "concurrent spawns create unique children", %{deps: deps, profile: profile} do - params = %{ - "task_description" => "Concurrent task", - "success_criteria" => "Complete", - "immediate_context" => "Test", - "approach_guidance" => "Standard", - "profile" => profile.name - } - - deps_with_mock = - Map.merge(deps, %{ - dynsup_fn: fn _pid, _config, _opts -> - pid = spawn(fn -> :timer.sleep(:infinity) end) - track_pid(deps, pid) - {:ok, pid} - end - }) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - - # Spawn 5 children sequentially (avoid Task.async complexity) - results = - for _ <- 1..5 do - Spawn.execute(params, "parent-1", opts) - end - - # All should succeed - child_ids = - Enum.map(results, fn {:ok, result} -> - result.agent_id - end) - - # All IDs should be unique - assert length(Enum.uniq(child_ids)) == 5, - "Expected 5 unique child IDs, got #{length(Enum.uniq(child_ids))}" - - # Wait for all background tasks to complete before cleanup - Enum.each(child_ids, fn child_id -> - wait_for_spawn_complete(child_id) - end) - - # Cleanup handled by on_exit via pids_tracker - end - end - describe "R11: Full Spawn Flow System Test" do # R11: WHEN user creates task with spawn_child action # THEN child appears in UI and responds @@ -527,6 +393,7 @@ defmodule Quoracle.Actions.SpawnAsyncTest do Map.merge(deps, %{ dynsup_fn: fn _pid, config, _opts -> agent_id = config.agent_id + mock_pid = self() # Spawn child that registers ITSELF (not the background Task) child_pid = @@ -538,17 +405,19 @@ defmodule Quoracle.Actions.SpawnAsyncTest do parent_id: parent_id }) - # Notify test that child is alive + # Notify mock that registration is complete, then notify test + send(mock_pid, :child_registered_internal) send(test_pid, {:child_alive, child_spawned_ref, agent_id}) :timer.sleep(:infinity) end) track_pid(deps, child_pid) - # Give child time to register + # Wait for child to actually register (event-based, not time-based) receive do + :child_registered_internal -> :ok after - 50 -> :ok + 5000 -> :ok end {:ok, child_pid} @@ -586,42 +455,4 @@ defmodule Quoracle.Actions.SpawnAsyncTest do # Cleanup handled by on_exit via pids_tracker end end - - describe "return format compatibility" do - # Verify async spawn returns same format as current sync spawn - test "async spawn returns compatible format with sync spawn", %{deps: deps, profile: profile} do - params = %{ - "task_description" => "Format test", - "success_criteria" => "Complete", - "immediate_context" => "Test", - "approach_guidance" => "Standard", - "profile" => profile.name - } - - deps_with_mock = - Map.put(deps, :dynsup_fn, fn _pid, _config, _opts -> - pid = spawn_link(fn -> :timer.sleep(:infinity) end) - track_pid(deps, pid) - {:ok, pid} - end) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - {:ok, result} = Spawn.execute(params, "parent-1", opts) - - # Required fields - assert result.action == "spawn" - assert is_binary(result.agent_id) - assert %DateTime{} = result.spawned_at - - # In async pattern, pid may not be immediately available - # But message field should indicate async status - if Map.has_key?(result, :message) do - assert result.message =~ "background" - end - - # Wait for background task to complete before test cleanup - wait_for_spawn_complete(result.agent_id) - # Cleanup handled by on_exit via pids_tracker - end - end end diff --git a/test/quoracle/actions/spawn_budget_test.exs b/test/quoracle/actions/spawn_budget_test.exs index 94ca381..20e9366 100644 --- a/test/quoracle/actions/spawn_budget_test.exs +++ b/test/quoracle/actions/spawn_budget_test.exs @@ -322,7 +322,7 @@ defmodule Quoracle.Actions.SpawnBudgetTest do @tag :integration test "R31: N/A parent can spawn child with budget", %{deps: deps, profile: profile} do # Create parent with N/A budget (unlimited) - parent_budget = %{mode: :na, allocated: nil, committed: nil} + parent_budget = Quoracle.Budget.Schema.new_na() {:ok, parent_pid} = spawn_parent_with_budget(deps, parent_budget) {:ok, parent_state} = Core.get_state(parent_pid) @@ -384,7 +384,7 @@ defmodule Quoracle.Actions.SpawnBudgetTest do # R57: Error Message Contains Guidance [INTEGRATION] # The :budget_required error, when surfaced through Spawn.execute, should # return a descriptive string message (not just the atom) that guides the - # LLM on what to do: include "budget" param and use "get_budget" to check funds. + # LLM on what to do: include "budget" param @tag :r57 @tag :integration test "R57: budget_required error message guides LLM", %{deps: deps, profile: profile} do @@ -421,11 +421,9 @@ defmodule Quoracle.Actions.SpawnBudgetTest do # Assert: Error contains guidance for the LLM agent # The error should be a string message (not just atom) that mentions: # 1. "budget" - what's required - # 2. "get_budget" - how to check available funds assert {:error, message} = result assert is_binary(message), "Error should be a descriptive string, got: #{inspect(message)}" assert message =~ "budget", "Error message should mention 'budget'" - assert message =~ "get_budget", "Error message should mention 'get_budget' action" end # NOTE: R53 (N/A parent unchanged), R54 (nil budget unchanged), R55 (root + budget OK), diff --git a/test/quoracle/actions/spawn_test.exs b/test/quoracle/actions/spawn_test.exs index d985e6e..a530817 100644 --- a/test/quoracle/actions/spawn_test.exs +++ b/test/quoracle/actions/spawn_test.exs @@ -503,29 +503,6 @@ defmodule Quoracle.Actions.SpawnTest do assert payload.task == "Test broadcast" end - test "includes timestamp in spawn broadcast", %{deps: deps, profile: profile} do - params = %{ - "task_description" => "Test", - "success_criteria" => "Complete", - "immediate_context" => "Test", - "approach_guidance" => "Standard", - "profile" => profile.name - } - - deps_with_mock = - Map.put(deps, :dynsup_fn, fn _pid, _config, _opts -> - {:ok, spawn_link(fn -> :timer.sleep(:infinity) end)} - end) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - {:ok, result} = Spawn.execute(params, "parent-1", opts) - child_pid = wait_for_spawn_complete(result.agent_id) - if child_pid, do: Process.exit(child_pid, :kill) - - assert_receive {:agent_spawned, payload}, 30_000 - assert %DateTime{} = payload.timestamp - end - test "no broadcast on spawn failure", %{deps: deps, profile: profile} do mock_dynsup = fn _pid, _config, _opts -> {:error, :spawn_error} @@ -556,29 +533,6 @@ defmodule Quoracle.Actions.SpawnTest do # No agent_spawned broadcast on failure refute_receive {:agent_spawned, _}, 100 end - - test "broadcasts to correct pubsub topic", %{deps: deps, profile: profile} do - params = %{ - "task_description" => "Topic test", - "success_criteria" => "Complete", - "immediate_context" => "Test", - "approach_guidance" => "Standard", - "profile" => profile.name - } - - deps_with_mock = - Map.put(deps, :dynsup_fn, fn _pid, _config, _opts -> - {:ok, spawn_link(fn -> :timer.sleep(:infinity) end)} - end) - - opts = Map.to_list(deps_with_mock) ++ [agent_pid: self()] - {:ok, result} = Spawn.execute(params, "parent-1", opts) - child_pid = wait_for_spawn_complete(result.agent_id) - if child_pid, do: Process.exit(child_pid, :kill) - - # Should receive on agents:lifecycle topic - assert_receive {:agent_spawned, _payload}, 30_000 - end end # Property-Based Tests diff --git a/test/quoracle/actions/todo_integration_test.exs b/test/quoracle/actions/todo_integration_test.exs index 2a9a93a..17ad8b7 100644 --- a/test/quoracle/actions/todo_integration_test.exs +++ b/test/quoracle/actions/todo_integration_test.exs @@ -99,59 +99,6 @@ defmodule Quoracle.Actions.TodoIntegrationTest do end end - describe "field name enforcement" do - test "prevents LLM from using alternative field names" do - # All these variations should be rejected - invalid_variations = [ - # Wrong content field names - %{"items" => [%{"task" => "Test", "state" => "todo"}]}, - %{"items" => [%{"description" => "Test", "state" => "todo"}]}, - %{"items" => [%{"title" => "Test", "state" => "todo"}]}, - %{"items" => [%{"text" => "Test", "state" => "todo"}]}, - %{"items" => [%{"item" => "Test", "state" => "todo"}]}, - %{"items" => [%{"details" => "Test", "state" => "todo"}]}, - # Wrong state field names - %{"items" => [%{"content" => "Test", "status" => "todo"}]}, - %{"items" => [%{"content" => "Test", "progress" => "todo"}]}, - %{"items" => [%{"content" => "Test", "phase" => "todo"}]}, - %{"items" => [%{"content" => "Test", "stage" => "todo"}]} - ] - - for invalid_params <- invalid_variations do - action_json = %{ - "action" => "todo", - "params" => invalid_params - } - - assert {:error, :missing_required_field} = Validator.validate_action(action_json), - "Should reject params: #{inspect(invalid_params)}" - end - end - - test "accepts only the exact schema structure" do - # Only this exact structure should pass - valid_params = %{ - "items" => [ - %{"content" => "First task", "state" => "todo"}, - %{"content" => "Second task", "state" => "pending"}, - %{"content" => "Third task", "state" => "done"} - ] - } - - action_json = %{ - "action" => "todo", - "params" => valid_params - } - - assert {:ok, validated} = Validator.validate_action(action_json) - assert length(validated.params.items) == 3 - - assert Enum.all?(validated.params.items, fn item -> - Map.keys(item) -- [:content, :state] == [] - end) - end - end - describe "LLM response validation" do test "validates realistic LLM response format" do # Actual format an LLM would generate diff --git a/test/quoracle/actions/todo_test.exs b/test/quoracle/actions/todo_test.exs index e9567ee..899565f 100644 --- a/test/quoracle/actions/todo_test.exs +++ b/test/quoracle/actions/todo_test.exs @@ -79,38 +79,6 @@ defmodule Quoracle.Actions.TodoTest do assert result.count == 2 end - test "preserves order of TODO items", %{ - pubsub: pubsub, - agent_pid: agent_pid, - agent_id: agent_id - } do - items = for i <- 1..10, do: %{content: "Task #{i}", state: :todo} - params = %{items: items} - opts = [pubsub: pubsub, agent_pid: agent_pid] - - assert {:ok, result} = Todo.execute(params, agent_id, opts) - assert result.count == 10 - end - - test "allows multiple items with same state", %{ - pubsub: pubsub, - agent_pid: agent_pid, - agent_id: agent_id - } do - params = %{ - items: [ - %{content: "First todo", state: :todo}, - %{content: "Second todo", state: :todo}, - %{content: "Third todo", state: :todo} - ] - } - - opts = [pubsub: pubsub, agent_pid: agent_pid] - - assert {:ok, result} = Todo.execute(params, agent_id, opts) - assert result.count == 3 - end - test "handles very long content strings", %{ pubsub: pubsub, agent_pid: agent_pid, diff --git a/test/quoracle/actions/wait_broadcast_test.exs b/test/quoracle/actions/wait_broadcast_test.exs index 20e8eab..9d71653 100644 --- a/test/quoracle/actions/wait_broadcast_test.exs +++ b/test/quoracle/actions/wait_broadcast_test.exs @@ -146,33 +146,6 @@ defmodule Quoracle.Actions.WaitBroadcastTest do end end - test "includes correct action metadata in broadcasts", %{agent_id: agent_id, pubsub: pubsub} do - router = spawn_wait_router(agent_id, pubsub) - - on_exit(fn -> - if Process.alive?(router), do: GenServer.stop(router, :normal, :infinity) - end) - - # Execute wait through router - params = %{wait: 0.1} - - capture_log(fn -> - Router.execute(router, :wait, params, agent_id) - end) - - assert_receive {:action_started, payload}, 30_000 - - # Verify metadata structure - assert Map.has_key?(payload, :agent_id) - assert Map.has_key?(payload, :action_type) - assert Map.has_key?(payload, :action_id) - assert Map.has_key?(payload, :params) - assert Map.has_key?(payload, :timestamp) - - assert payload.action_type == :wait - assert payload.params == params - end - test "broadcasts preserve execution context for concurrent waits", %{ agent_id: agent_id, pubsub: pubsub @@ -200,37 +173,6 @@ defmodule Quoracle.Actions.WaitBroadcastTest do assert payload1.action_id != payload2.action_id assert payload1.agent_id == payload2.agent_id end - - test "broadcasts include duration in completed event", %{agent_id: agent_id, pubsub: pubsub} do - router = spawn_wait_router(agent_id, pubsub) - - on_exit(fn -> - if Process.alive?(router), do: GenServer.stop(router, :normal, :infinity) - end) - - # Execute wait with known duration through router - # Keep under smart_threshold for sync execution - params = %{wait: 0.05} - - capture_log(fn -> - send(self(), {:result, Router.execute(router, :wait, params, agent_id)}) - end) - - assert_received {:result, {:ok, _result}} - - # Should receive completed event with correct structure - assert_receive {:action_started, start_payload}, 30_000 - assert_receive {:action_completed, complete_payload}, 30_000 - - # Verify broadcast payloads have correct structure - assert start_payload.agent_id == agent_id - assert start_payload.params.wait == 0.05 - assert complete_payload.agent_id == agent_id - assert match?({:ok, _}, complete_payload.result) - - # The actual timing is tested in wait_test.exs with mock delay_fn - # Here we only care about broadcast behavior, not wall-clock time - end end describe "async wait broadcasts" do diff --git a/test/quoracle/actions/wait_enhanced_test.exs b/test/quoracle/actions/wait_enhanced_test.exs index 03f6861..b5506fe 100644 --- a/test/quoracle/actions/wait_enhanced_test.exs +++ b/test/quoracle/actions/wait_enhanced_test.exs @@ -115,26 +115,6 @@ defmodule Quoracle.Actions.WaitEnhancedTest do end describe "timer management" do - test "delivers timer expiry message to AGENT_Core", %{deps: deps} do - capture_log(fn -> - # Register as agent - Registry.register(deps.registry, {:agent, "timer_agent"}, %{}) - - # Start async wait - {:ok, response} = - Wait.execute(%{wait: 0.1}, "timer_agent", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - ) - - timer_id = response.timer_id - - # Should receive timer expiry message (generous timeout for system under load) - assert_receive {:wait_expired, ^timer_id}, 5000 - end) - end - test "cancels previous timer when new wait starts", %{deps: deps} do # Register as agent Registry.register(deps.registry, {:agent, "cancel_agent"}, %{}) @@ -219,41 +199,6 @@ defmodule Quoracle.Actions.WaitEnhancedTest do end end - describe "error handling" do - test "returns error for negative duration", %{deps: deps} do - capture_log(fn -> - result = - Wait.execute(%{wait: -1}, "test_agent", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - ) - - assert {:error, :invalid_wait_value} = result - end) - end - - test "handles unregistered agents with async mode", %{deps: deps} do - # Register the process to enable async mode - Registry.register(deps.registry, {:agent, "unregistered_agent"}, %{}) - - {:ok, response} = - Wait.execute(%{wait: 0.05}, "unregistered_agent", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - ) - - # Should use async mode when registered - assert response.async == true - assert response.timer_id - - # Verify timer expiry message (generous timeout for system under load) - timer_id = response.timer_id - assert_receive {:wait_expired, ^timer_id}, 5000 - end - end - describe "integration with ACTION_Router" do test "router detects async response and handles accordingly", %{deps: deps} do alias Quoracle.Actions.Router diff --git a/test/quoracle/actions/wait_interruption_test.exs b/test/quoracle/actions/wait_interruption_test.exs deleted file mode 100644 index 3ab73ae..0000000 --- a/test/quoracle/actions/wait_interruption_test.exs +++ /dev/null @@ -1,214 +0,0 @@ -defmodule Quoracle.Actions.WaitInterruptionTest do - @moduledoc """ - Integration tests for wait action interruption behavior. - Tests verify that wait action with timer support will be interruptible. - """ - - use ExUnit.Case, async: true - import ExUnit.CaptureLog - import Test.IsolationHelpers - - alias Quoracle.Actions.Wait - - setup do - # Create isolated dependencies for testing - deps = create_isolated_deps() - {:ok, deps: deps} - end - - describe "wait action timer behavior" do - test "wait action with numeric value should return timer_id for interruption", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-agent-1"}, %{}) - - # Execute wait action with 5-second timer - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: 5}, "wait-agent-1", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - assert wait_result.action == "wait" - # Should return timer_id for ConsensusHandler to store - assert is_reference(wait_result.timer_id) - assert wait_result.async == true - - # Timer message should arrive after 5 seconds (we won't wait that long) - refute_receive {:wait_expired, _}, 100 - end - - test "wait action with true should not return timer_id (indefinite wait)", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-agent-2"}, %{}) - - # Execute wait action with indefinite wait - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: true}, "wait-agent-2", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - assert wait_result.action == "wait" - # Should NOT have timer_id for indefinite wait - refute Map.has_key?(wait_result, :timer_id) - assert wait_result.async == true - - # No timer to expire - refute_receive {:wait_expired, _}, 100 - end - - test "wait action with false should not create timer (immediate continuation)", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-agent-3"}, %{}) - - # Execute wait action with immediate continuation - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: false}, "wait-agent-3", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - assert wait_result.action == "wait" - # Should NOT have timer_id for immediate continuation - refute Map.has_key?(wait_result, :timer_id) - assert wait_result.async == false - - # No timer to expire - refute_receive {:wait_expired, _}, 100 - end - - test "wait timer expiry sends correct message", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-agent-4"}, %{}) - - # Execute wait action with timer (500ms for reliable timing) - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: 0.5}, "wait-agent-4", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - assert is_reference(wait_result.timer_id) - - # Wait for timer to expire (3x margin for scheduler variance) - timer_id = wait_result.timer_id - assert_receive {:wait_expired, ^timer_id}, 1500 - end - - test "different wait values create different timer behaviors", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-agent-5"}, %{}) - - # Test multiple wait values - test_cases = [ - # indefinite wait - {true, :no_timer, true}, - # immediate continuation - {false, :no_timer, false}, - # zero wait - {0, :no_timer, false}, - # 1 second wait - {1, :has_timer, true}, - # 50ms wait - {0.05, :has_timer, true} - ] - - for {wait_value, timer_expectation, async_expected} <- test_cases do - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: wait_value}, "wait-agent-5", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, result}} - assert result.action == "wait" - assert result.async == async_expected - - case timer_expectation do - :has_timer -> - assert is_reference(result.timer_id) - - :no_timer -> - refute Map.has_key?(result, :timer_id) - end - end - end - end - - describe "wait action return values for interruption support" do - test "timed wait returns metadata for timer storage", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-storage-1"}, %{}) - - # Execute wait action with timer - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: 1}, "wait-storage-1", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - # Should return timer_id that ConsensusHandler can store - assert is_reference(wait_result.timer_id) - assert wait_result.async == true - end - - test "wait action timer can coexist with wait parameter timer concept", %{deps: deps} do - Registry.register(deps.registry, {:agent, "wait-storage-2"}, %{}) - - # Execute wait action - capture_log(fn -> - send( - self(), - {:result, - Wait.execute(%{wait: 0.5}, "wait-storage-2", - registry: deps.registry, - pubsub: deps.pubsub, - agent_pid: self() - )} - ) - end) - - assert_received {:result, {:ok, wait_result}} - action_timer_ref = wait_result.timer_id - assert is_reference(action_timer_ref) - - # The timer will fire after 500ms (use default timeout for CI stability) - assert_receive {:wait_expired, ^action_timer_ref}, 30_000 - end - end -end diff --git a/test/quoracle/actions/wait_isolation_test.exs b/test/quoracle/actions/wait_isolation_test.exs index 1ada6cd..53a044f 100644 --- a/test/quoracle/actions/wait_isolation_test.exs +++ b/test/quoracle/actions/wait_isolation_test.exs @@ -204,33 +204,5 @@ defmodule Quoracle.Actions.WaitIsolationTest do assert_receive {:action_error, payload}, 30_000 assert payload.agent_id == agent_id end - - test "duration limit errors broadcast to isolated PubSub", %{pubsub: pubsub} do - agent_id = "test-agent-limit" - - # Spawn per-action Router (v28.0) - {:ok, router} = spawn_wait_router(agent_id, pubsub) - - on_exit(fn -> - if Process.alive?(router) do - try do - GenServer.stop(router, :normal, :infinity) - catch - :exit, _ -> :ok - end - end - end) - - # Test with a negative duration to trigger an error - capture_log(fn -> - send(self(), {:result, Router.execute(router, :wait, %{wait: -1}, agent_id)}) - end) - - assert_received {:result, {:error, _}} - - # Should receive error broadcast in isolated PubSub - assert_receive {:action_error, payload}, 30_000 - assert payload.agent_id == agent_id - end end end diff --git a/test/quoracle/actions/wait_unification_test.exs b/test/quoracle/actions/wait_unification_test.exs index 19d3382..3fa8c57 100644 --- a/test/quoracle/actions/wait_unification_test.exs +++ b/test/quoracle/actions/wait_unification_test.exs @@ -6,25 +6,12 @@ defmodule Quoracle.Actions.WaitUnificationTest do - ACTION_Schema v18.0: wait action uses 'wait' parameter (not 'duration') - ACTION_Validator v7.0: union type validation for boolean and number """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Actions.Schema alias Quoracle.Actions.Validator describe "ACTION_Schema v18.0 - Wait Parameter Unification" do - # R13 from CONSENSUS_PromptBuilder spec - test "[UNIT] wait action schema uses wait parameter (not duration)" do - {:ok, schema} = Schema.get_schema(:wait) - - # Verify 'wait' is in optional params - assert :wait in schema.optional_params, - "Expected :wait in optional_params, got: #{inspect(schema.optional_params)}" - - # Verify 'duration' is NOT in optional params (breaking change) - refute :duration in schema.optional_params, - "Expected :duration to be removed, but found in: #{inspect(schema.optional_params)}" - end - # R14 from CONSENSUS_PromptBuilder spec test "[UNIT] wait action schema accepts boolean and number types" do {:ok, schema} = Schema.get_schema(:wait) @@ -34,22 +21,6 @@ defmodule Quoracle.Actions.WaitUnificationTest do "Expected {:union, [:boolean, :number]}, got: #{inspect(schema.param_types[:wait])}" end - test "[UNIT] wait action has correct consensus rule for wait parameter" do - {:ok, schema} = Schema.get_schema(:wait) - - # Verify consensus rule exists for wait (not duration) - assert Map.has_key?(schema.consensus_rules, :wait), - "Expected consensus_rules to have :wait key" - - # Verify consensus rule is percentile 50 (median) - assert schema.consensus_rules.wait == {:percentile, 50}, - "Expected {:percentile, 50}, got: #{inspect(schema.consensus_rules.wait)}" - - # Verify no consensus rule for duration (removed) - refute Map.has_key?(schema.consensus_rules, :duration), - "Expected consensus_rules to NOT have :duration key" - end - # R15 from CONSENSUS_PromptBuilder spec test "[UNIT] wait action description emphasizes equivalence with wait parameter" do description = Schema.get_action_description(:wait) @@ -62,19 +33,6 @@ defmodule Quoracle.Actions.WaitUnificationTest do String.contains?(description, "parameter"), "Expected description to mention unification with wait parameter, got: #{description}" end - - # R16 from CONSENSUS_PromptBuilder spec - test "[UNIT] wait action schema does not include duration parameter" do - {:ok, schema} = Schema.get_schema(:wait) - - # Check param_types doesn't have duration - refute Map.has_key?(schema.param_types, :duration), - "Expected param_types to NOT have :duration key" - - # Check param_descriptions doesn't have duration - refute Map.has_key?(schema.param_descriptions || %{}, :duration), - "Expected param_descriptions to NOT have :duration key" - end end describe "ACTION_Validator v7.0 - Boolean Type Support" do @@ -215,90 +173,4 @@ defmodule Quoracle.Actions.WaitUnificationTest do assert {:error, _reason} = Validator.validate_action(action) end end - - describe "Union Type Validation" do - test "[UNIT] validator handles union type {:union, [:boolean, :number]}" do - # This tests the generic union type validation capability - # that powers the wait parameter validation - # Note: This will fail until union type support is implemented - - # Test that wait action with boolean is validated correctly - action_bool = %{ - "action" => "wait", - "params" => %{"wait" => true}, - "reasoning" => "Testing union type with boolean" - } - - # Test that wait action with number is validated correctly - action_num = %{ - "action" => "wait", - "params" => %{"wait" => 42}, - "reasoning" => "Testing union type with number" - } - - # Both should pass when union type support is implemented - assert {:ok, _} = Validator.validate_action(action_bool), - "Expected boolean to be valid for union type" - - assert {:ok, _} = Validator.validate_action(action_num), - "Expected number to be valid for union type" - - # String should fail - action_str = %{ - "action" => "wait", - "params" => %{"wait" => "invalid"}, - "reasoning" => "Testing union type with invalid string" - } - - assert {:error, _} = Validator.validate_action(action_str), - "Expected string to be invalid for union type" - end - end - - describe "Integration Tests" do - test "[INTEGRATION] wait action with various values through full validation" do - test_cases = [ - {true, "Wait indefinitely", :ok}, - {false, "Continue immediately", :ok}, - {0, "Zero wait", :ok}, - {5, "Wait 5 seconds", :ok}, - {0.1, "Wait 100ms", :ok}, - {"5", "String value", :error}, - {nil, "Nil value", :error}, - {[], "List value", :error} - ] - - for {wait_value, reasoning, expected} <- test_cases do - action = %{ - "action" => "wait", - "params" => %{"wait" => wait_value}, - "reasoning" => reasoning - } - - case expected do - :ok -> - assert {:ok, validated} = Validator.validate_action(action), - "Expected validation to pass for wait=#{inspect(wait_value)}" - - assert validated.params.wait == wait_value - - :error -> - assert {:error, _} = Validator.validate_action(action), - "Expected validation to fail for wait=#{inspect(wait_value)}" - end - end - end - - test "[INTEGRATION] empty params for wait action still valid" do - # Wait action should allow empty params (all params are optional) - action = %{ - "action" => "wait", - "params" => %{}, - "reasoning" => "Default wait behavior" - } - - assert {:ok, validated} = Validator.validate_action(action) - assert validated.params == %{} - end - end end diff --git a/test/quoracle/agent/AGENTS.md b/test/quoracle/agent/AGENTS.md index df4ee18..bcf51c1 100644 --- a/test/quoracle/agent/AGENTS.md +++ b/test/quoracle/agent/AGENTS.md @@ -75,6 +75,17 @@ - Integration (R205-R206): Sequential completion flow — last SC triggers, shell deferred for batch_sync - System (R207): End-to-end with real Core GenServer, stale check_id deferred until batch_sync completes +## System Prompt Cache Tests (Added 2026-02-22) +- consensus/system_prompt_cache_test.exs: 10 tests (R1-R9), 523 lines, async: true + - Unit (R1): State.new cached_system_prompt field default + - Integration (R2-R3): Lazy build on first consensus, reuse on subsequent + - Unit (R4): learn_skills invalidation + - Integration (R5): Rebuild after invalidation reflects new skills + - Unit (R6): Cached prompt matches fresh PromptBuilder build + - Integration (R7): Fast-path (Option E) bypasses cache entirely + - Integration (R8): Multi-model uniformity (3-model pool) + - Integration (R9, regression): field_prompts content included in cached prompt (2 tests) + ## Action Executor Regression Tests (Added 2026-02-14) - action_executor_regressions_test.exs: 15 tests (R1-R14 + R5b), 1060 lines, async: true - Bug 1 (error stall): R1, R2, R3, R4 — always-sync error + wait:true continues consensus @@ -83,6 +94,14 @@ - TestActionHandler: R12 — shell_routers keyed by command_id not action_id - System: R13 (failed spawn recovery), R14 (shell + check_id round-trip) +## HTTP Action Timeout Override Tests (Added 2026-02-23) +- consensus_handler/action_executor_http_timeout_test.exs: 5 tests (R78-R82), async: true + - Unit (R78): answer_engine returns real result, not async tuple (120_000ms override) + - Unit (R79): fetch_web returns real result, not async tuple (60_000ms override) + - Unit (R80): call_api returns real result, not async tuple (120_000ms override) + - Unit (R81): generate_images returns real result, not async tuple (300_000ms override) + - Unit (R82): existing call_mcp/adjust_budget overrides preserved + ## Removed Files core_injection_test.exs, config_manager_injection_test.exs, dyn_sup_injection_test.exs (redundant after isolation) diff --git a/test/quoracle/agent/action_deadlock_prevention_test.exs b/test/quoracle/agent/action_deadlock_prevention_test.exs index 25b126c..a2b7217 100644 --- a/test/quoracle/agent/action_deadlock_prevention_test.exs +++ b/test/quoracle/agent/action_deadlock_prevention_test.exs @@ -142,15 +142,17 @@ defmodule Quoracle.Agent.ActionDeadlockPreventionTest do _result_state = ActionExecutor.execute_consensus_action(raw_state, action_response, agent_pid) - # CRITICAL: Core should respond within 200ms even though action takes 500ms. - # (200ms allows for scheduler pressure under parallel test load.) - # If ActionExecutor blocked synchronously, get_state would also block. + # CRITICAL: Core should respond well under 500ms even though action takes 500ms. + # (400ms threshold allows for scheduler pressure under heavy parallel test load + # while still clearly distinguishing from the 500ms+ response time that would + # indicate Core is blocked/deadlocked during action execution.) + # If ActionExecutor blocked synchronously, get_state would also block for 500ms+. start_time = System.monotonic_time(:millisecond) assert {:ok, _current_state} = Core.get_state(agent_pid) elapsed = System.monotonic_time(:millisecond) - start_time - assert elapsed < 200, - "Core took #{elapsed}ms to respond - should be < 200ms. " <> + assert elapsed < 400, + "Core took #{elapsed}ms to respond - should be < 400ms. " <> "This suggests Core is blocked during action execution (deadlock)." end) end diff --git a/test/quoracle/agent/action_executor_regressions_test.exs b/test/quoracle/agent/action_executor_regressions_test.exs index e7f3b6c..d721989 100644 --- a/test/quoracle/agent/action_executor_regressions_test.exs +++ b/test/quoracle/agent/action_executor_regressions_test.exs @@ -964,10 +964,10 @@ defmodule Quoracle.Agent.ActionExecutorRegressionsTest do {:action_result, action_id, {:error, :invalid_budget_format}, result_opts} ) - # Wait for result to be processed (pending_actions cleared) + # Wait for result to be processed (pending_actions cleared AND consensus scheduled) post_state = wait_for_condition(agent_pid, fn state -> - map_size(state.pending_actions) == 0 + map_size(state.pending_actions) == 0 and state.consensus_scheduled end) |> extract_state() diff --git a/test/quoracle/agent/config_manager_model_histories_test.exs b/test/quoracle/agent/config_manager_model_histories_test.exs index 4065029..9db6622 100644 --- a/test/quoracle/agent/config_manager_model_histories_test.exs +++ b/test/quoracle/agent/config_manager_model_histories_test.exs @@ -6,7 +6,7 @@ defmodule Quoracle.Agent.ConfigManagerModelHistoriesTest do Tests R1-R7 from AGENT_ConfigManager_PerModelHistories.md spec. """ # Use DataCase for R5 integration test (needs DB access) - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.ConfigManager alias Quoracle.Consensus.Manager diff --git a/test/quoracle/agent/consensus/per_model_query_ace_injector_test.exs b/test/quoracle/agent/consensus/per_model_query_ace_injector_test.exs index 69fc5a6..11e9e7f 100644 --- a/test/quoracle/agent/consensus/per_model_query_ace_injector_test.exs +++ b/test/quoracle/agent/consensus/per_model_query_ace_injector_test.exs @@ -10,10 +10,9 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryAceInjectorTest do ARC Verification Criteria: R61-R64 """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.Consensus.PerModelQuery - alias Quoracle.Agent.ConsensusHandler.AceInjector # ========== TEST HELPERS ========== @@ -126,266 +125,5 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryAceInjectorTest do assert all_content =~ "", "Model state should be injected into messages" assert all_content =~ "Task is 75% complete" end - - test "AceInjector.inject_ace_context callable directly" do - # Unit test for isolated AceInjector behavior - state = make_state_with_ace("test-model", [make_lesson("Test lesson")]) - messages = [%{role: "user", content: "Test"}] - - result = AceInjector.inject_ace_context(state, messages, "test-model") - assert is_list(result) - end - end - - # ========== R62: ACE INJECTED BEFORE TODOS ========== - - describe "R62: ACE in first message, todos in last" do - test "ACE appears in first user message, not last" do - model_id = "test-model" - lessons = [make_lesson("Historical knowledge")] - - state = - make_state_with_ace(model_id, lessons) - |> Map.put(:todos, [%{content: "Current task", state: :todo}]) - - # Build messages as PerModelQuery would - messages = build_messages_with_injections(state, model_id) - - # Find first user message - first_user_idx = Enum.find_index(messages, &(&1.role == "user")) - assert first_user_idx != nil - - first_user = Enum.at(messages, first_user_idx) - - # First user message should have ACE content - assert first_user.content =~ "" - assert first_user.content =~ "Historical knowledge" - - # Last message should have todos - last = Enum.at(messages, -1) - assert last.content =~ "" - - # If first user != last, ACE should only be in first user message - # If first user == last (single user message), both are in same message - if first_user.content != last.content do - refute last.content =~ "", "ACE should not be in last message" - end - end - - test "injection order correct with multiple messages" do - model_id = "test-model" - lessons = [make_lesson("Lesson content")] - - state = - make_state_with_ace(model_id, lessons) - |> Map.put(:model_histories, %{ - model_id => [ - make_history_entry(:user, "First user"), - make_history_entry(:assistant, "First response"), - make_history_entry(:user, "Second user"), - make_history_entry(:assistant, "Second response"), - make_history_entry(:user, "Third user") - ] - }) - |> Map.put(:todos, [%{content: "Task", state: :todo}]) - - messages = build_messages_with_injections(state, model_id) - - # First user message has ACE - first_user = Enum.find(messages, &(&1.role == "user")) - assert first_user.content =~ "" - - # Last message has todos - last = Enum.at(messages, -1) - assert last.content =~ "" - - # They should be in different messages - first_user_content = first_user.content - last_content = last.content - - # If first == last, both should be present - if first_user_content == last_content do - assert first_user_content =~ "" - assert first_user_content =~ "" - else - # Otherwise, ACE only in first, todos only in last - refute last_content =~ "" - end - end - end - - # ========== R63: MODEL-SPECIFIC ACE ========== - - describe "R63: each model receives its own lessons" do - test "model A gets only model A lessons" do - model_a = "anthropic:claude-sonnet-4" - model_b = "google:gemini-2.0-flash" - - state = %{ - agent_id: "test-agent", - task_id: "test-task", - model_histories: %{ - model_a => [make_history_entry(:user, "Hello from A")], - model_b => [make_history_entry(:user, "Hello from B")] - }, - context_lessons: %{ - model_a => [make_lesson("Lesson for A")], - model_b => [make_lesson("Lesson for B")] - }, - model_states: %{}, - todos: [], - children: [], - budget_data: nil - } - - messages_a = build_messages_with_injections(state, model_a) - messages_b = build_messages_with_injections(state, model_b) - - # Model A should see only its lessons - content_a = Enum.map_join(messages_a, " ", & &1.content) - assert content_a =~ "Lesson for A" - refute content_a =~ "Lesson for B" - - # Model B should see only its lessons - content_b = Enum.map_join(messages_b, " ", & &1.content) - assert content_b =~ "Lesson for B" - refute content_b =~ "Lesson for A" - end - - test "model-specific state also isolated" do - model_a = "model-a" - model_b = "model-b" - - state = %{ - agent_id: "test-agent", - task_id: "test-task", - model_histories: %{ - model_a => [make_history_entry(:user, "Hello")], - model_b => [make_history_entry(:user, "Hello")] - }, - context_lessons: %{}, - model_states: %{ - model_a => make_model_state("State for A"), - model_b => make_model_state("State for B") - }, - todos: [], - children: [], - budget_data: nil - } - - messages_a = build_messages_with_injections(state, model_a) - messages_b = build_messages_with_injections(state, model_b) - - content_a = Enum.map_join(messages_a, " ", & &1.content) - content_b = Enum.map_join(messages_b, " ", & &1.content) - - assert content_a =~ "State for A" - refute content_a =~ "State for B" - - assert content_b =~ "State for B" - refute content_b =~ "State for A" - end - end - - # ========== R64: EMPTY ACE NO CHANGE ========== - - describe "R64: empty ACE leaves messages unchanged" do - test "no injection when no lessons for model" do - model_id = "test-model" - other_model = "other-model" - - state = %{ - agent_id: "test-agent", - task_id: "test-task", - model_histories: %{ - model_id => [make_history_entry(:user, "Original content")] - }, - # Lessons only for other model - context_lessons: %{other_model => [make_lesson("Other lesson")]}, - model_states: %{}, - todos: [], - children: [], - budget_data: nil - } - - messages = build_messages_with_injections(state, model_id) - - # No ACE tags should be present - content = Enum.map_join(messages, " ", & &1.content) - refute content =~ "" - refute content =~ "" - refute content =~ "Other lesson" - end - - test "no injection when context_lessons empty" do - model_id = "test-model" - - state = %{ - agent_id: "test-agent", - task_id: "test-task", - model_histories: %{ - model_id => [make_history_entry(:user, "Just a message")] - }, - context_lessons: %{}, - model_states: %{}, - todos: [], - children: [], - budget_data: nil - } - - messages = build_messages_with_injections(state, model_id) - - content = Enum.map_join(messages, " ", & &1.content) - refute content =~ "" - refute content =~ "" - - # Original content preserved - assert content =~ "Just a message" - end - - test "no injection when both lessons and state nil for model" do - model_id = "test-model" - - state = %{ - agent_id: "test-agent", - task_id: "test-task", - model_histories: %{ - model_id => [make_history_entry(:user, "User content")] - }, - context_lessons: nil, - model_states: nil, - todos: [], - children: [], - budget_data: nil - } - - messages = build_messages_with_injections(state, model_id) - - content = Enum.map_join(messages, " ", & &1.content) - refute content =~ "" - refute content =~ "" - end - end - - # ========== HELPER: BUILD MESSAGES WITH ALL INJECTIONS ========== - - # Simulates the injection order in PerModelQuery.query_single_model_with_retry/3 - defp build_messages_with_injections(state, model_id) do - alias Quoracle.Agent.ContextManager - alias Quoracle.Agent.ConsensusHandler.{TodoInjector, ChildrenInjector} - - # 1. Build base messages from history - messages = ContextManager.build_conversation_messages(state, model_id) - - # 2. Inject ACE into FIRST user message (NEW - what we're testing) - messages = AceInjector.inject_ace_context(state, messages, model_id) - - # 3. Inject todos into LAST message - messages = TodoInjector.inject_todo_context(state, messages) - - # 4. Inject children into LAST message - messages = ChildrenInjector.inject_children_context(state, messages) - - messages end end diff --git a/test/quoracle/agent/consensus/per_model_query_dynamic_max_tokens_test.exs b/test/quoracle/agent/consensus/per_model_query_dynamic_max_tokens_test.exs index 2eb4537..fec00b4 100644 --- a/test/quoracle/agent/consensus/per_model_query_dynamic_max_tokens_test.exs +++ b/test/quoracle/agent/consensus/per_model_query_dynamic_max_tokens_test.exs @@ -118,7 +118,7 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do }} end - opts = [model_query_fn: mock_query_fn, test_mode: true] + opts = [model_query_fn: mock_query_fn, test_mode: true, force_token_management: true] # This should calculate dynamic max_tokens using the formula # max_tokens = min(context_window - input_tokens, output_limit) @@ -174,7 +174,7 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do }} end - opts = [model_query_fn: mock_query_fn, test_mode: true] + opts = [model_query_fn: mock_query_fn, test_mode: true, force_token_management: true] # calculate_max_tokens doesn't exist yet - the query will still use # the old static max_tokens: 4096 default until implemented @@ -263,7 +263,8 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do :ets.insert(condensation_triggered, {:triggered, true}) {:ok, %{lessons: [], state: []}} end, - test_mode: true + test_mode: true, + force_token_management: true ] # When available_output < 4096, proactive condensation should trigger @@ -352,7 +353,8 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do reflector_fn: fn _messages, _model_id, _opts -> {:ok, %{lessons: [], state: []}} end, - test_mode: true + test_mode: true, + force_token_management: true ] # If condensation happens, messages should be rebuilt from condensed state @@ -442,7 +444,8 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do reflector_fn: fn _messages, _model_id, _opts -> {:ok, %{lessons: [], state: []}} end, - test_mode: true + test_mode: true, + force_token_management: true ] # Query should succeed even when context is tight @@ -536,7 +539,8 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do reflector_fn: fn _messages, _model_id, _opts -> {:ok, %{lessons: [], state: []}} end, - test_mode: true + test_mode: true, + force_token_management: true ] {:ok, _response, _result_state} = @@ -630,7 +634,8 @@ defmodule Quoracle.Agent.Consensus.PerModelQueryDynamicMaxTokensTest do reflector_fn: fn _messages, _model_id, _opts -> {:ok, %{lessons: [], state: []}} end, - test_mode: true + test_mode: true, + force_token_management: true ] # After condensation, the max_tokens passed to query should reflect diff --git a/test/quoracle/agent/consensus/system_prompt_cache_test.exs b/test/quoracle/agent/consensus/system_prompt_cache_test.exs new file mode 100644 index 0000000..93e61a0 --- /dev/null +++ b/test/quoracle/agent/consensus/system_prompt_cache_test.exs @@ -0,0 +1,523 @@ +defmodule Quoracle.Agent.Consensus.SystemPromptCacheTest do + @moduledoc """ + Tests for CACHE_SystemPrompt - System Prompt Caching. + + Verifies the system prompt caching mechanism works correctly: + cache lifecycle (build, reuse, invalidate, rebuild), consistency + with uncached builds, compatibility with Option E fast-path, + and multi-model uniformity. + + WorkGroupID: feat-20260222-system-prompt-cache + Packet: 1 (Single Packet) + + ARC Verification Criteria: R1-R9 + """ + use Quoracle.DataCase, async: true + + alias Quoracle.Agent.Core + alias Quoracle.Agent.Core.State + alias Quoracle.Consensus.PromptBuilder + + # ========== TEST HELPERS ========== + + # Builds a valid orient action JSON response for mock consensus + defp orient_response do + Jason.encode!(%{ + "action" => "orient", + "params" => %{ + "current_situation" => "Processing", + "goal_clarity" => "Clear", + "available_resources" => "Available", + "key_challenges" => "None", + "delegation_consideration" => "none" + }, + "reasoning" => "Test orient response", + "wait" => true + }) + end + + # Creates a mock model_query_fn that captures messages and returns an orient response. + # Sends {:query_messages, model_id, messages} to the test process for each query. + defp capturing_model_query_fn(test_pid) do + fn messages, [model_id], _opts -> + send(test_pid, {:query_messages, model_id, messages}) + + {:ok, + %{ + successful_responses: [%{model: model_id, content: orient_response()}], + failed_models: [] + }} + end + end + + # Creates an agent config suitable for spawning with consensus pipeline support. + # The model_query_fn injects a spy to capture messages sent during consensus. + defp agent_config(deps, opts) do + test_pid = Keyword.get(opts, :test_pid, self()) + model_pool = Keyword.get(opts, :model_pool, ["test-model-1"]) + active_skills = Keyword.get(opts, :active_skills, []) + + %{ + agent_id: "cache-test-#{System.unique_integer([:positive])}", + test_mode: true, + model_pool: model_pool, + model_histories: Map.new(model_pool, fn m -> {m, []} end), + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub, + active_skills: active_skills, + # model_query_fn forces real consensus pipeline (not fast-path) + test_opts: [model_query_fn: capturing_model_query_fn(test_pid)] + } + end + + # Triggers consensus by sending a user message and waits for action completion. + # Returns the system prompt(s) captured from the model_query_fn spy. + defp trigger_consensus_and_capture(agent_pid, model_pool \\ ["test-model-1"]) do + # Send a user message to trigger consensus + Core.handle_message(agent_pid, "Test message for consensus") + + # Capture system prompts from each model query + Enum.map(model_pool, fn model_id -> + assert_receive {:query_messages, ^model_id, messages}, 5000 + system_msg = Enum.find(messages, &(&1.role == "system")) + {model_id, system_msg && system_msg.content} + end) + end + + # ========== R1: STATE FIELD DEFAULT [UNIT] ========== + + describe "R1: Cache field exists in State [UNIT]" do + test "State.new includes cached_system_prompt field defaulting to nil" do + state = + State.new(%{ + agent_id: "test-agent", + registry: :test_registry, + dynsup: self(), + pubsub: :test_pubsub + }) + + assert Map.has_key?(state, :cached_system_prompt), + "State struct must include cached_system_prompt field" + + assert state.cached_system_prompt == nil, + "cached_system_prompt must default to nil" + end + end + + # ========== R2: LAZY CACHE BUILD [INTEGRATION] ========== + + describe "R2: Cache built lazily on first consensus [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self()), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "cache built lazily on first consensus cycle", %{agent_pid: agent_pid} do + # Before consensus, cache should be nil + {:ok, state_before} = Core.get_state(agent_pid) + assert state_before.cached_system_prompt == nil + + # Trigger consensus (sends message, which triggers consensus pipeline) + _captured = trigger_consensus_and_capture(agent_pid) + + # Wait for action to complete (orient with wait:true just returns) + # Use GenServer.call to synchronize state + {:ok, state_after} = Core.get_state(agent_pid) + + assert is_binary(state_after.cached_system_prompt), + "cached_system_prompt should be a non-nil string after first consensus, " <> + "got: #{inspect(state_after.cached_system_prompt)}" + + assert String.length(state_after.cached_system_prompt) > 0, + "cached_system_prompt should be a non-empty string" + end + end + + # ========== R3: CACHE REUSE [INTEGRATION] ========== + + describe "R3: Cache reused on subsequent consensus [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self()), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "cache reused without rebuild on subsequent consensus", %{agent_pid: agent_pid} do + # First consensus: builds and caches prompt + [{_model, prompt_1}] = trigger_consensus_and_capture(agent_pid) + + # Wait for first consensus to settle + {:ok, state_after_1} = Core.get_state(agent_pid) + cached_after_1 = state_after_1.cached_system_prompt + + assert is_binary(cached_after_1), "Cache should be populated after first consensus" + + # Second consensus: should reuse the same cached prompt + [{_model, prompt_2}] = trigger_consensus_and_capture(agent_pid) + + {:ok, state_after_2} = Core.get_state(agent_pid) + cached_after_2 = state_after_2.cached_system_prompt + + # The system prompt sent to the model should be identical both times + assert prompt_1 == prompt_2, + "System prompt should be identical on second consensus (reused from cache)" + + # The cached value should be the same reference/string + assert cached_after_1 == cached_after_2, + "Cached system prompt should remain unchanged between consensus cycles" + end + end + + # ========== R4: LEARN_SKILLS INVALIDATION [UNIT] ========== + + describe "R4: learn_skills invalidates cached system prompt [UNIT]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self()), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "learn_skills invalidates cached system prompt", %{agent_pid: agent_pid} do + # First, populate the cache by running consensus + _captured = trigger_consensus_and_capture(agent_pid) + + {:ok, state_with_cache} = Core.get_state(agent_pid) + + assert is_binary(state_with_cache.cached_system_prompt), + "Cache should be populated after consensus" + + # Cast learn_skills to the agent + skill_metadata = [ + %{ + name: "test_skill", + permanent: true, + loaded_at: DateTime.utc_now(), + description: "A test skill", + path: "/tmp/test_skill", + metadata: %{} + } + ] + + GenServer.cast(agent_pid, {:learn_skills, skill_metadata}) + + # Synchronize with a GenServer.call to ensure the cast was processed + {:ok, state_after_learn} = Core.get_state(agent_pid) + + assert state_after_learn.cached_system_prompt == nil, + "cached_system_prompt should be nil after learn_skills" + + # Verify active_skills were updated + assert state_after_learn.active_skills != [], + "active_skills should be updated after learn_skills" + end + end + + # ========== R5: REBUILD AFTER INVALIDATION [INTEGRATION] ========== + + describe "R5: Cache rebuilt after learn_skills invalidation [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self()), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "cache rebuilt after learn_skills invalidation reflects new skills", %{ + agent_pid: agent_pid + } do + # First consensus: cache with no active skills + [{_model, _prompt_before}] = trigger_consensus_and_capture(agent_pid) + {:ok, state_1} = Core.get_state(agent_pid) + assert is_binary(state_1.cached_system_prompt) + + # Learn a new skill (invalidates cache) + skill_metadata = [ + %{ + name: "newly_learned_skill", + permanent: true, + loaded_at: DateTime.utc_now(), + description: "A newly learned skill for cache test", + path: "/tmp/newly_learned_skill", + metadata: %{} + } + ] + + GenServer.cast(agent_pid, {:learn_skills, skill_metadata}) + {:ok, state_invalidated} = Core.get_state(agent_pid) + assert state_invalidated.cached_system_prompt == nil + + # Second consensus: should rebuild cache with new skills + [{_model, _prompt_after}] = trigger_consensus_and_capture(agent_pid) + {:ok, state_2} = Core.get_state(agent_pid) + + assert is_binary(state_2.cached_system_prompt), + "Cache should be rebuilt after invalidation" + + # The rebuilt cache should differ from the original (new skills included) + # Note: This assertion depends on PromptBuilder including skills in the prompt. + # If skills affect the prompt, the strings should differ. + assert state_2.cached_system_prompt != state_1.cached_system_prompt, + "Rebuilt cache should differ from original (new skills added)" + end + end + + # ========== R6: CACHE CONSISTENCY [UNIT] ========== + + describe "R6: Cache consistency with fresh build [UNIT]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self()), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "cached prompt matches fresh build with same inputs", %{agent_pid: agent_pid} do + # Run consensus to populate the cache + _captured = trigger_consensus_and_capture(agent_pid) + + {:ok, state} = Core.get_state(agent_pid) + cached = state.cached_system_prompt + assert is_binary(cached) + + # Build a fresh prompt with the same inputs + fresh_opts = [ + profile_name: state.profile_name, + profile_description: state.profile_description, + capability_groups: state.capability_groups, + active_skills: state.active_skills, + skills_path: Map.get(state, :skills_path), + field_prompts: %{system_prompt: state.system_prompt}, + sandbox_owner: state.sandbox_owner + ] + + fresh = PromptBuilder.build_system_prompt_with_context(fresh_opts) + + assert cached == fresh, + "Cached prompt should be identical to a fresh build with the same inputs" + end + end + + # ========== R7: FAST-PATH BYPASS [INTEGRATION] ========== + + describe "R7: Fast-path bypasses cache entirely [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + # Spawn agent WITHOUT model_query_fn so fast-path activates + # (test_mode: true + no simulate flags + no model_query_fn = fast path) + config = %{ + agent_id: "fast-path-#{System.unique_integer([:positive])}", + test_mode: true, + model_pool: ["test-model-1"], + model_histories: %{"test-model-1" => []}, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub + # Intentionally NO test_opts with model_query_fn + } + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + config, + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid} + end + + test "fast-path consensus does not build or use cache", %{agent_pid: agent_pid} do + # Verify cache starts as nil + {:ok, state_before} = Core.get_state(agent_pid) + assert state_before.cached_system_prompt == nil + + # Send a message to trigger consensus (fast-path will handle it) + Core.handle_message(agent_pid, "Test message for fast path") + + # Give the agent time to process (fast path returns mock orient with wait:true) + # Synchronize with GenServer.call + {:ok, state_after} = Core.get_state(agent_pid) + + # Cache should still be nil because fast-path skips the entire pipeline + assert state_after.cached_system_prompt == nil, + "Fast-path should not build or populate the system prompt cache" + end + end + + # ========== R9: FIELD_PROMPTS INCLUDED IN CACHE [INTEGRATION] ========== + # Regression: Cache build must include field_prompts so agents with role/cognitive_style + # get those XML tags embedded in the identity section of the system prompt. + + describe "R9: Cached prompt includes field_prompts content [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + # Agent with a non-nil system_prompt (simulates field-based role/cognitive_style) + config = + agent_config(deps, test_pid: self()) + |> Map.put(:system_prompt, "Cache Regression Test Role") + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + config, + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, deps: deps} + end + + test "system prompt sent to models contains agent field_prompts content", %{ + agent_pid: agent_pid + } do + # Trigger consensus and capture the system prompt sent to the model + [{_model, system_prompt}] = trigger_consensus_and_capture(agent_pid) + + assert is_binary(system_prompt), "Model should receive a system prompt" + + # The system prompt must include the agent's role from field_prompts + assert system_prompt =~ "Cache Regression Test Role", + "Cached system prompt must include field_prompts content (role/cognitive_style). " <> + "Got prompt of length #{String.length(system_prompt)} without the expected role tag." + end + + test "cached prompt matches fresh build when agent has field_prompts", %{ + agent_pid: agent_pid + } do + # Run consensus to populate the cache + _captured = trigger_consensus_and_capture(agent_pid) + + {:ok, state} = Core.get_state(agent_pid) + cached = state.cached_system_prompt + assert is_binary(cached) + + # Build a fresh prompt with the SAME inputs including field_prompts + fresh_opts = [ + profile_name: state.profile_name, + profile_description: state.profile_description, + capability_groups: state.capability_groups, + active_skills: state.active_skills, + skills_path: Map.get(state, :skills_path), + field_prompts: %{system_prompt: state.system_prompt}, + sandbox_owner: state.sandbox_owner + ] + + fresh = PromptBuilder.build_system_prompt_with_context(fresh_opts) + + assert cached == fresh, + "Cached prompt must match fresh build with field_prompts. " <> + "Cached length: #{String.length(cached)}, fresh length: #{String.length(fresh)}. " <> + "Cached includes role? #{String.contains?(cached, "Cache Regression")}. " <> + "Fresh includes role? #{String.contains?(fresh, "Cache Regression")}." + end + end + + # ========== R8: MULTI-MODEL UNIFORMITY [INTEGRATION] ========== + + describe "R8: All models use same cached prompt [INTEGRATION]" do + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + _profile = create_test_profile() + + model_pool = ["model-a", "model-b", "model-c"] + + {:ok, agent_pid} = + spawn_agent_with_cleanup( + deps.dynsup, + agent_config(deps, test_pid: self(), model_pool: model_pool), + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner + ) + + {:ok, agent_pid: agent_pid, model_pool: model_pool} + end + + test "all models in consensus pool receive same cached system prompt", %{ + agent_pid: agent_pid, + model_pool: model_pool + } do + # Trigger consensus across 3 models + captured = trigger_consensus_and_capture(agent_pid, model_pool) + + # Extract system prompts from each model's messages + system_prompts = + Enum.map(captured, fn {_model_id, prompt} -> + assert is_binary(prompt), + "Each model should receive a system prompt" + + prompt + end) + + # All 3 should be identical + [first | rest] = system_prompts + + Enum.each(rest, fn prompt -> + assert prompt == first, + "All models should receive identical system prompt from cache" + end) + + # Verify the cache is populated + {:ok, state} = Core.get_state(agent_pid) + + assert is_binary(state.cached_system_prompt), + "Cache should be populated after multi-model consensus" + end + end +end diff --git a/test/quoracle/agent/consensus_continuation_handler_v2_test.exs b/test/quoracle/agent/consensus_continuation_handler_v2_test.exs deleted file mode 100644 index 6f7962e..0000000 --- a/test/quoracle/agent/consensus_continuation_handler_v2_test.exs +++ /dev/null @@ -1,212 +0,0 @@ -defmodule Quoracle.Agent.ConsensusContinuationHandlerV2Test do - @moduledoc """ - Tests for ConsensusContinuationHandler v2.0 (fix-20251209-035351 Packet 3). - - R1: handle_wait_timeout delegates to ConsensusHandler.get_action_consensus - R2: does NOT call ContextManager.build_conversation_messages - R3: only handles timer cleanup and delegation - """ - - use ExUnit.Case, async: true - - alias Quoracle.Agent.ConsensusContinuationHandler - - defp base_state do - %{ - agent_id: "test-agent-#{System.unique_integer([:positive])}", - model_histories: %{}, - pubsub: nil, - wait_timer: nil, - test_mode: true - } - end - - describe "[UNIT] handle_wait_timeout delegation (R1)" do - test "R1: delegates to ConsensusHandler.get_action_consensus" do - # R1: handle_wait_timeout should delegate directly to ConsensusHandler - # NOT use injected get_consensus_fn parameter - - state = base_state() - timer_id = "timer-#{System.unique_integer([:positive])}" - test_pid = self() - - # Mock cancel_timer_fn - just returns state - cancel_timer_fn = fn s -> s end - - # Mock execute_action_fn - capture execution - execute_action_fn = fn s, action -> - send(test_pid, {:executed, action}) - s - end - - # R1: The v2.0 signature should NOT require get_consensus_fn - # It should delegate to ConsensusHandler.get_action_consensus(state) internally - # This test will FAIL if implementation still requires 5 args with get_consensus_fn - - # v2.0 signature: handle_wait_timeout(state, timer_id, cancel_timer_fn, execute_action_fn) - # Old signature: handle_wait_timeout(state, timer_id, cancel_timer_fn, get_consensus_fn, execute_action_fn) - - result = - ConsensusContinuationHandler.handle_wait_timeout( - state, - timer_id, - cancel_timer_fn, - execute_action_fn - ) - - # Should return {:noreply, state} tuple - assert {:noreply, _new_state} = result - end - - test "R1: handle_wait_timeout uses ConsensusHandler directly" do - # This test verifies that handle_wait_timeout calls ConsensusHandler - # internally rather than accepting a get_consensus_fn parameter - - state = base_state() - timer_id = "timer-#{System.unique_integer([:positive])}" - - cancel_timer_fn = fn s -> s end - execute_action_fn = fn s, _action -> s end - - # Try calling with 4 args (v2.0 pattern without get_consensus_fn) - # Will fail if implementation requires 5 args - result = - ConsensusContinuationHandler.handle_wait_timeout( - state, - timer_id, - cancel_timer_fn, - execute_action_fn - ) - - assert {:noreply, _} = result - end - end - - describe "[UNIT] no ContextManager calls (R2)" do - test "R2: does not call ContextManager.build_conversation_messages" do - # R2: The handler should NOT call ContextManager.build_conversation_messages - # ConsensusHandler handles all context building internally - - state = base_state() - timer_id = "timer-test" - - cancel_timer_fn = fn s -> s end - execute_action_fn = fn s, _action -> s end - - # If implementation calls ContextManager.build_conversation_messages, - # the state would need more fields. With minimal state, it should still work - # because ConsensusHandler handles context building (not this handler) - - result = - ConsensusContinuationHandler.handle_wait_timeout( - state, - timer_id, - cancel_timer_fn, - execute_action_fn - ) - - # Should succeed without needing full conversation context - assert {:noreply, _} = result - end - end - - describe "[UNIT] single responsibility (R3)" do - test "R3: handler only manages timer cleanup and delegates" do - # R3: ConsensusContinuationHandler should: - # 1. Cancel the timer (via cancel_timer_fn) - # 2. Delegate to ConsensusHandler.get_action_consensus - # 3. Execute action (via execute_action_fn) - # Nothing else - no message building, no context manipulation - - state = base_state() - timer_id = "timer-cleanup" - test_pid = self() - - # Track that cancel_timer_fn is called - cancel_timer_fn = fn s -> - send(test_pid, :timer_cancelled) - s - end - - # Track that execute_action_fn is called with action - execute_action_fn = fn s, action -> - send(test_pid, {:action_executed, action}) - s - end - - _result = - ConsensusContinuationHandler.handle_wait_timeout( - state, - timer_id, - cancel_timer_fn, - execute_action_fn - ) - - # Verify timer was cancelled - assert_receive :timer_cancelled - # Verify action was executed (means consensus was obtained) - assert_receive {:action_executed, action}, 30_000 - assert is_map(action) - end - - test "R3: handler adds wait_timeout to history before consensus" do - # The handler should add the wait_timeout event to history - # so the LLM knows the timer expired - - state = base_state() - timer_id = "timer-history" - test_pid = self() - - cancel_timer_fn = fn s -> s end - - execute_action_fn = fn s, _action -> - # Capture state to verify history was updated - send(test_pid, {:state_at_execute, s}) - s - end - - _result = - ConsensusContinuationHandler.handle_wait_timeout( - state, - timer_id, - cancel_timer_fn, - execute_action_fn - ) - - # The state passed to execute should have history updated - assert_receive {:state_at_execute, updated_state}, 30_000 - # model_histories should have been updated with wait_timeout event - histories = Map.get(updated_state, :model_histories, %{}) - - # model_histories should exist and be a map (wait_timeout event recorded in history) - assert Map.has_key?(updated_state, :model_histories) - assert is_map(histories) - end - end - - describe "[UNIT] handle_consensus_continuation delegation" do - test "handle_consensus_continuation uses ConsensusHandler directly" do - # handle_consensus_continuation should also delegate to ConsensusHandler - # without requiring a request_consensus_fn parameter - - state = base_state() - test_pid = self() - - execute_action_fn = fn s, action -> - send(test_pid, {:executed, action}) - s - end - - # v2.0: handle_consensus_continuation(state, execute_action_fn) - # Old: handle_consensus_continuation(state, request_consensus_fn, execute_action_fn) - - result = - ConsensusContinuationHandler.handle_consensus_continuation( - state, - execute_action_fn - ) - - assert {:noreply, _} = result - end - end -end diff --git a/test/quoracle/agent/consensus_continuation_handler_v5_test.exs b/test/quoracle/agent/consensus_continuation_handler_v5_test.exs deleted file mode 100644 index 2593112..0000000 --- a/test/quoracle/agent/consensus_continuation_handler_v5_test.exs +++ /dev/null @@ -1,97 +0,0 @@ -defmodule Quoracle.Agent.ConsensusContinuationHandlerV5Test do - @moduledoc """ - Tests for ConsensusContinuationHandler v5.0 - Delegation to StateUtils.cancel_wait_timer - - WorkGroupID: fix-20260117-consensus-staleness - Packet: 1 (Foundation) - Requirements: R17-R18 - """ - use ExUnit.Case, async: true - - alias Quoracle.Agent.ConsensusContinuationHandler - alias Quoracle.Agent.StateUtils - - describe "[UNIT] R17: Delegates to StateUtils" do - test "ConsensusContinuationHandler.cancel_wait_timer delegates to StateUtils" do - # Test that both produce identical results (delegation behavior) - timer_ref = Process.send_after(self(), :delegate_test, 60_000) - state = %{wait_timer: {timer_ref, :timed_wait}} - - # ConsensusContinuationHandler should delegate to StateUtils - # Both calls should produce the same result - cch_result = ConsensusContinuationHandler.cancel_wait_timer(state) - - # Create fresh timer for StateUtils call - timer_ref2 = Process.send_after(self(), :delegate_test2, 60_000) - state2 = %{wait_timer: {timer_ref2, :timed_wait}} - su_result = StateUtils.cancel_wait_timer(state2) - - # Both should clear wait_timer to nil - assert cch_result.wait_timer == nil - assert su_result.wait_timer == nil - end - - test "ConsensusContinuationHandler.cancel_wait_timer handles nil like StateUtils" do - state = %{wait_timer: nil} - - cch_result = ConsensusContinuationHandler.cancel_wait_timer(state) - su_result = StateUtils.cancel_wait_timer(state) - - assert cch_result == su_result - assert cch_result == state - end - - test "ConsensusContinuationHandler.cancel_wait_timer handles 2-tuple like StateUtils" do - timer_ref = Process.send_after(self(), :two_tuple_delegate, 60_000) - state = %{wait_timer: {timer_ref, :timed_wait}} - - result = ConsensusContinuationHandler.cancel_wait_timer(state) - - assert result.wait_timer == nil - end - - test "ConsensusContinuationHandler.cancel_wait_timer handles 3-tuple like StateUtils" do - timer_ref = Process.send_after(self(), :three_tuple_delegate, 60_000) - state = %{wait_timer: {timer_ref, "timer-id", 1}} - - result = ConsensusContinuationHandler.cancel_wait_timer(state) - - assert result.wait_timer == nil - end - end - - describe "[INTEGRATION] R18: Backward Compatibility" do - test "existing cancel_wait_timer callers work unchanged" do - # Create state with 2-tuple timer (existing format) - timer_ref = Process.send_after(self(), :compat_test, 60_000) - state = %{wait_timer: {timer_ref, :timed_wait}, agent_id: "test"} - - # Old calling pattern should still work - result = ConsensusContinuationHandler.cancel_wait_timer(state) - - assert result.wait_timer == nil - assert result.agent_id == "test" - end - - test "handle_wait_timeout uses cancel_wait_timer correctly" do - # The cancel_timer_fn callback receives state and should clear timer - timer_ref = Process.send_after(self(), :timeout_test, 60_000) - - state = %{ - wait_timer: {timer_ref, :timed_wait}, - agent_id: "compat-agent", - model_histories: %{"model1" => []}, - skip_auto_consensus: true - } - - # Simulate the cancel_timer_fn that Core passes - cancel_timer_fn = &ConsensusContinuationHandler.cancel_wait_timer/1 - - # Apply the function - result = cancel_timer_fn.(state) - - assert result.wait_timer == nil - assert result.agent_id == "compat-agent" - end - end -end diff --git a/test/quoracle/agent/consensus_handler/action_executor_http_timeout_test.exs b/test/quoracle/agent/consensus_handler/action_executor_http_timeout_test.exs new file mode 100644 index 0000000..5d21b43 --- /dev/null +++ b/test/quoracle/agent/consensus_handler/action_executor_http_timeout_test.exs @@ -0,0 +1,274 @@ +defmodule Quoracle.Agent.ConsensusHandler.ActionExecutorHttpTimeoutTest do + @moduledoc """ + Regression tests for HTTP action timeout overrides in ActionExecutor. + + Root cause: answer_engine, fetch_web, call_api, and generate_images make + HTTP API calls that routinely exceed the 100ms smart_threshold. Without + an explicit timeout override, Execution.execute_action enters "smart mode", + yields for only 100ms, and returns {:async_task, ...}. The ActionExecutor + background Task casts that opaque tuple as the "result", the agent removes + the action from pending_actions, and the *real* result arriving later is + silently discarded (unknown action_id). + + Fix (v39.0): Timeout overrides for all HTTP-based actions in ActionExecutor: + :answer_engine → 120_000ms + :fetch_web → 60_000ms + :call_api → 120_000ms + :generate_images → 300_000ms + + Test strategy: Dispatch each action through ActionExecutor. The actions + will fail quickly with specific errors (no API key, connection refused, + etc.) but those are real errors returned synchronously — NOT the opaque + {:async, ref, ack} tuple that caused the silent-discard bug. + + Tests: + - R78: answer_engine gets 120_000ms timeout [UNIT] + - R79: fetch_web gets 60_000ms timeout [UNIT] + - R80: call_api gets 120_000ms timeout [UNIT] + - R81: generate_images gets 300_000ms timeout [UNIT] + - R82: existing call_mcp/adjust_budget overrides preserved [UNIT] + """ + + use Quoracle.DataCase, async: true + + alias Quoracle.Agent.Core + alias Quoracle.Agent.ConsensusHandler.ActionExecutor + alias Quoracle.Profiles.CapabilityGroups + alias Quoracle.Tasks.Task, as: TaskSchema + alias Test.IsolationHelpers + + import Test.AgentTestHelpers + + @moduletag capture_log: true + + @all_capability_groups CapabilityGroups.groups() + + setup %{sandbox_owner: sandbox_owner} do + deps = IsolationHelpers.create_isolated_deps() + deps = Map.put(deps, :sandbox_owner, sandbox_owner) + + {:ok, task} = + Repo.insert(%TaskSchema{ + id: Ecto.UUID.generate(), + prompt: "http timeout override test", + status: "running" + }) + + {:ok, deps: deps, task: task} + end + + defp spawn_agent(deps, task) do + agent_id = "http-timeout-#{System.unique_integer([:positive])}" + + config = %{ + agent_id: agent_id, + task_id: task.id, + test_mode: true, + skip_auto_consensus: true, + sandbox_owner: deps.sandbox_owner, + pubsub: deps.pubsub, + capability_groups: @all_capability_groups, + spawn_complete_notify: self(), + prompt_fields: %{ + provided: %{task_description: "HTTP timeout test"}, + injected: %{global_context: "", constraints: []}, + transformed: %{} + }, + models: [] + } + + spawn_agent_with_cleanup(deps.dynsup, config, + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: deps.sandbox_owner + ) + end + + defp build_test_state(agent_pid) do + {:ok, state} = Core.get_state(agent_pid) + %{state | pending_actions: %{}, action_counter: 0} + end + + # Helper: dispatch action and assert the result is a real error (synchronous path), + # not the opaque {:async, ref, ack} that indicates smart mode broke the result chain. + defp assert_sync_result(state, action_response, action_name) do + _dispatched = ActionExecutor.execute_consensus_action(state, action_response, self()) + + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + # The result must NOT be the async tuple — that's the whole bug + refute match?({:async, _, _}, result), + "#{action_name} returned {:async, ...} — timeout override not applied, " <> + "result will be silently discarded" + + # The result must NOT be :timeout — that would mean the timeout was set + # but too short (shouldn't happen for fast-failing actions) + refute match?({:error, :timeout}, result), + "#{action_name} returned {:error, :timeout} — unexpected" + + # It should be a real error (no API key, connection refused, etc.) + result + after + 15_000 -> + flunk("No action result within 15s for #{action_name}") + end + end + + # ============================================================================ + # R78: answer_engine gets 120_000ms timeout [UNIT] + # + # WHEN ActionExecutor dispatches :answer_engine + # THEN the action executes synchronously (timeout override bypasses smart mode) + # AND returns a real error (not {:async, ref, ack}) + # ============================================================================ + + describe "R78: answer_engine timeout override" do + @tag :r78 + @tag :unit + test "answer_engine returns real result, not async tuple", + %{deps: deps, task: task} do + {:ok, agent_pid} = spawn_agent(deps, task) + state = build_test_state(agent_pid) + + action_response = %{ + action: :answer_engine, + params: %{prompt: "test query for timeout regression"}, + wait: false, + reasoning: "Testing answer_engine timeout override" + } + + result = assert_sync_result(state, action_response, "answer_engine") + + # Should fail with a real error (no Gemini API key configured in test) + assert match?({:error, _}, result), + "Expected an error (no API key), got: #{inspect(result)}" + end + end + + # ============================================================================ + # R79: fetch_web gets 60_000ms timeout [UNIT] + # + # WHEN ActionExecutor dispatches :fetch_web + # THEN the action executes synchronously (timeout override bypasses smart mode) + # AND returns a real error (not {:async, ref, ack}) + # ============================================================================ + + describe "R79: fetch_web timeout override" do + @tag :r79 + @tag :unit + test "fetch_web returns real result, not async tuple", + %{deps: deps, task: task} do + {:ok, agent_pid} = spawn_agent(deps, task) + state = build_test_state(agent_pid) + + action_response = %{ + action: :fetch_web, + params: %{url: "http://localhost:1/nonexistent"}, + wait: false, + reasoning: "Testing fetch_web timeout override" + } + + result = assert_sync_result(state, action_response, "fetch_web") + + # Should fail with connection refused (localhost:1 is not listening) + assert match?({:error, _}, result), + "Expected an error (connection refused), got: #{inspect(result)}" + end + end + + # ============================================================================ + # R80: call_api gets 120_000ms timeout [UNIT] + # + # WHEN ActionExecutor dispatches :call_api + # THEN the action executes synchronously (timeout override bypasses smart mode) + # AND returns a real error (not {:async, ref, ack}) + # ============================================================================ + + describe "R80: call_api timeout override" do + @tag :r80 + @tag :unit + test "call_api returns real result, not async tuple", + %{deps: deps, task: task} do + {:ok, agent_pid} = spawn_agent(deps, task) + state = build_test_state(agent_pid) + + action_response = %{ + action: :call_api, + params: %{api_type: "rest", url: "http://localhost:1/nonexistent", method: "GET"}, + wait: false, + reasoning: "Testing call_api timeout override" + } + + result = assert_sync_result(state, action_response, "call_api") + + # Should fail with connection refused + assert match?({:error, _}, result), + "Expected an error (connection refused), got: #{inspect(result)}" + end + end + + # ============================================================================ + # R81: generate_images gets 300_000ms timeout [UNIT] + # + # WHEN ActionExecutor dispatches :generate_images + # THEN the action executes synchronously (timeout override bypasses smart mode) + # AND returns a real error (not {:async, ref, ack}) + # ============================================================================ + + describe "R81: generate_images timeout override" do + @tag :r81 + @tag :unit + test "generate_images returns real result, not async tuple", + %{deps: deps, task: task} do + {:ok, agent_pid} = spawn_agent(deps, task) + state = build_test_state(agent_pid) + + action_response = %{ + action: :generate_images, + params: %{prompt: "test image for timeout regression"}, + wait: false, + reasoning: "Testing generate_images timeout override" + } + + result = assert_sync_result(state, action_response, "generate_images") + + # Should fail with a real error (no image generation API configured in test) + assert match?({:error, _}, result), + "Expected an error (no API key), got: #{inspect(result)}" + end + end + + # ============================================================================ + # R82: existing timeout overrides preserved [UNIT] + # + # WHEN ActionExecutor dispatches :call_mcp or :adjust_budget + # THEN existing timeout overrides (600_000 and :infinity) are unchanged + # + # Regression: ensures adding new HTTP overrides didn't break existing ones. + # Verified by dispatching call_mcp — it should fail with a specific error + # (no MCP connection), NOT :timeout. + # ============================================================================ + + describe "R82: existing overrides preserved" do + @tag :r82 + @tag :unit + test "call_mcp still gets 600_000 timeout after HTTP action additions", + %{deps: deps, task: task} do + {:ok, agent_pid} = spawn_agent(deps, task) + state = build_test_state(agent_pid) + + action_response = %{ + action: :call_mcp, + params: %{connection_id: "nonexistent", tool_name: "test", arguments: %{}}, + wait: false, + reasoning: "Testing call_mcp timeout preserved" + } + + result = assert_sync_result(state, action_response, "call_mcp") + + # Should fail with MCP error, not timeout + refute match?({:error, :timeout}, result), + "call_mcp should not timeout. Got: #{inspect(result)}" + end + end +end diff --git a/test/quoracle/agent/consensus_handler/action_executor_timeout_test.exs b/test/quoracle/agent/consensus_handler/action_executor_timeout_test.exs new file mode 100644 index 0000000..febf200 --- /dev/null +++ b/test/quoracle/agent/consensus_handler/action_executor_timeout_test.exs @@ -0,0 +1,479 @@ +defmodule Quoracle.Agent.ConsensusHandler.ActionExecutorTimeoutTest do + @moduledoc """ + Tests for AGENT_ConsensusHandler v28.0 - ActionExecutor adjust_budget Timeout Override. + + WorkGroupID: fix-20260223-cost-display-budget-timeout + Packet: Packet 2 (Bug 2 — Budget Timeout) + + Root cause: adjust_budget is in @always_sync_actions, forcing explicit timeout path + with default 5000ms Task.yield. When the parent GenServer is blocked processing the + adjust_budget Task's GenServer.call, the 5s timer fires and kills the action. + + Fix: ActionExecutor adds timeout: :infinity for :adjust_budget (like :call_mcp gets 600_000). + + Tests: + - R73: adjust_budget completes when child busy [INTEGRATION] + - R74: adjust_budget gets :infinity timeout override [UNIT] — KEY failing test + - R75: call_mcp timeout unchanged at 600_000 [UNIT] + - R76: Other actions no timeout override [UNIT] + - R77: Parent responsive during adjust_budget [SYSTEM] + """ + + use Quoracle.DataCase, async: true + + alias Quoracle.Agent.Core + alias Quoracle.Agent.ConsensusHandler.ActionExecutor + alias Quoracle.Profiles.CapabilityGroups + alias Quoracle.Tasks.Task, as: TaskSchema + alias Test.IsolationHelpers + + import Test.AgentTestHelpers + + @moduletag capture_log: true + + # All capability groups (allows all actions) + @all_capability_groups CapabilityGroups.groups() + + setup %{sandbox_owner: sandbox_owner} do + deps = IsolationHelpers.create_isolated_deps() + deps = Map.put(deps, :sandbox_owner, sandbox_owner) + + # Create shared task for agents + {:ok, task} = + Repo.insert(%TaskSchema{ + id: Ecto.UUID.generate(), + prompt: "action executor timeout test", + status: "running" + }) + + # Subscribe to lifecycle events + Phoenix.PubSub.subscribe(deps.pubsub, "agents:lifecycle") + + {:ok, deps: deps, task: task} + end + + # Helper to spawn a parent agent with budget + defp spawn_parent_with_budget(deps, task, budget_data) do + agent_id = "parent-timeout-#{System.unique_integer([:positive])}" + + config = %{ + agent_id: agent_id, + task_id: task.id, + test_mode: true, + skip_auto_consensus: true, + sandbox_owner: deps.sandbox_owner, + pubsub: deps.pubsub, + budget_data: budget_data, + capability_groups: @all_capability_groups, + spawn_complete_notify: self(), + prompt_fields: %{ + provided: %{task_description: "Parent task"}, + injected: %{global_context: "", constraints: []}, + transformed: %{} + }, + models: [] + } + + spawn_agent_with_cleanup(deps.dynsup, config, + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: deps.sandbox_owner + ) + end + + # Helper to spawn a child agent under a parent + defp spawn_child_under_parent(deps, task, parent_pid, parent_state, child_budget_data) do + child_config = %{ + agent_id: "child-timeout-#{System.unique_integer([:positive])}", + task_id: task.id, + parent_id: parent_state.agent_id, + test_mode: true, + skip_auto_consensus: true, + sandbox_owner: deps.sandbox_owner, + pubsub: deps.pubsub, + budget_data: child_budget_data, + prompt_fields: %{ + provided: %{task_description: "Child task"}, + injected: %{global_context: "", constraints: []}, + transformed: %{} + }, + models: [] + } + + {:ok, child_pid} = + spawn_agent_with_cleanup(deps.dynsup, child_config, + registry: deps.registry, + pubsub: deps.pubsub, + sandbox_owner: deps.sandbox_owner + ) + + # Register child with parent + {:ok, child_state} = Core.get_state(child_pid) + + child_info = %{ + agent_id: child_state.agent_id, + spawned_at: DateTime.utc_now(), + budget_allocated: child_budget_data.allocated + } + + GenServer.cast(parent_pid, {:child_spawned, child_info}) + # Sync to ensure cast is processed + _ = Core.get_state(parent_pid) + + {:ok, child_pid, child_state} + end + + # ============================================================================ + # R74: ActionExecutor sets timeout :infinity for adjust_budget [UNIT] + # + # WHEN ActionExecutor builds execute_opts for :adjust_budget + # THEN timeout is :infinity in the opts passed to Router.execute + # + # Test strategy: Suspend the parent GenServer AFTER dispatching the action. + # The background task calls Core.adjust_child_budget(parent_id, ...) which + # does GenServer.call(parent_pid, ..., :infinity). With parent suspended, this + # call blocks indefinitely. The Router's Task.yield(task, timeout) behavior: + # - timeout=5000 (current): yields nil after 5s -> Task.shutdown -> {:error, :timeout} + # - timeout=:infinity (fix): yields :infinity -> never times out, we resume -> success + # + # Currently FAILS: no timeout override for adjust_budget -> 5s Task.yield -> :timeout + # ============================================================================ + + describe "R74: adjust_budget gets :infinity timeout" do + @tag :r74 + @tag :unit + test "adjust_budget does not time out at 5s when parent GenServer is busy", + %{deps: deps, task: task} do + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("30.00") + } + + {:ok, parent_pid} = spawn_parent_with_budget(deps, task, parent_budget) + {:ok, parent_state} = Core.get_state(parent_pid) + + child_budget = %{ + mode: :child, + allocated: Decimal.new("30.00"), + committed: Decimal.new("0") + } + + {:ok, _child_pid, child_state} = + spawn_child_under_parent(deps, task, parent_pid, parent_state, child_budget) + + # Build adjust_budget action + action_response = %{ + action: :adjust_budget, + params: %{child_id: child_state.agent_id, new_budget: "50.00"}, + wait: false, + reasoning: "Testing timeout override" + } + + {:ok, fresh_state} = Core.get_state(parent_pid) + + test_state = %{ + fresh_state + | pending_actions: %{}, + action_counter: 0 + } + + # Suspend parent GenServer. The background task's GenServer.call(parent_pid, ...) + # will block until we resume. This triggers the Router's Task.yield timeout: + # - Without fix: Task.yield(task, 5000) -> nil -> {:error, :timeout} + # - With fix: Task.yield(task, :infinity) -> never returns nil, waits for resume + :sys.suspend(parent_pid) + + # Dispatch through ActionExecutor — goes to Task.Supervisor background task + # which calls Router.execute -> Execution.execute_action -> Task.yield(task, timeout) + # The inner task calls AdjustBudget.execute -> Core.adjust_child_budget -> blocked call + _dispatched = + ActionExecutor.execute_consensus_action(test_state, action_response, self()) + + # Wait 7 seconds — longer than the 5s default timeout but well within :infinity. + # Without the fix: action result with {:error, :timeout} arrives at ~5s. + # With the fix: no result arrives (still waiting for parent to resume). + receive do + {:"$gen_cast", {:action_result, _action_id, {:error, :timeout}, _opts}} -> + # Resume parent for cleanup + :sys.resume(parent_pid) + + flunk( + "adjust_budget returned {:error, :timeout} after ~5s. " <> + "This proves ActionExecutor does NOT set timeout: :infinity for :adjust_budget. " <> + "The Router's Task.yield(task, 5000) killed the action." + ) + after + 7_000 -> + # No timeout error after 7s — the action is still waiting (timeout: :infinity) + # Resume parent so the action can complete + :sys.resume(parent_pid) + + # Now the GenServer.call unblocks and the action completes + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + assert match?({:ok, _}, result), + "After resume, adjust_budget should succeed. Got: #{inspect(result)}" + after + 10_000 -> + flunk("No action result after resuming parent. Action may have been killed.") + end + end + end + end + + # ============================================================================ + # R73: adjust_budget completes when child is busy [INTEGRATION] + # + # WHEN adjust_budget dispatched AND child agent is mid-consensus (busy) + # THEN action completes successfully (no timeout) + # + # This passes even without the fix because v3.0 BudgetHandler uses cast + # (not call) to the child, so the action completes fast regardless. + # Regression test: ensures the cast-based flow continues to work. + # ============================================================================ + + describe "R73: adjust_budget completes when child busy" do + @tag :r73 + @tag :integration + test "adjust_budget completes successfully even when child is busy", + %{deps: deps, task: task} do + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("30.00") + } + + {:ok, parent_pid} = spawn_parent_with_budget(deps, task, parent_budget) + {:ok, parent_state} = Core.get_state(parent_pid) + + child_budget = %{ + mode: :child, + allocated: Decimal.new("30.00"), + committed: Decimal.new("0") + } + + {:ok, child_pid, child_state} = + spawn_child_under_parent(deps, task, parent_pid, parent_state, child_budget) + + # Suspend child to simulate it being "busy" (mid-consensus) + :sys.suspend(child_pid) + + action_response = %{ + action: :adjust_budget, + params: %{child_id: child_state.agent_id, new_budget: "50.00"}, + wait: false, + reasoning: "Testing adjust while child busy" + } + + {:ok, fresh_state} = Core.get_state(parent_pid) + + test_state = %{ + fresh_state + | pending_actions: %{}, + action_counter: 0 + } + + _dispatched = + ActionExecutor.execute_consensus_action(test_state, action_response, self()) + + # v3.0 uses cast (not call) to child, so adjust_budget completes fast + # even with child suspended. Should complete well within 5 seconds. + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + assert match?({:ok, _}, result), + "adjust_budget should succeed even with child suspended. Got: #{inspect(result)}" + after + 5_000 -> + flunk("No action result within 5s. adjust_budget may have tried to call child.") + end + + # Resume child for cleanup + :sys.resume(child_pid) + end + end + + # ============================================================================ + # R75: call_mcp timeout unchanged at 600_000 [UNIT] + # + # WHEN ActionExecutor builds execute_opts for :call_mcp + # THEN timeout is 600_000 (unchanged by adjust_budget fix) + # + # Regression test: ensures the case statement preserves call_mcp behavior. + # Verified by dispatching :call_mcp and checking the Router sees the timeout. + # Since we don't have a real MCP server, the action will fail — but with a + # specific error (not :timeout), proving the 600_000ms timeout was applied. + # ============================================================================ + + describe "R75: call_mcp timeout unchanged" do + @tag :r75 + @tag :unit + test "call_mcp still gets 600_000 timeout, not :infinity", + %{deps: deps, task: task} do + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_parent_with_budget(deps, task, parent_budget) + {:ok, parent_state} = Core.get_state(parent_pid) + + action_response = %{ + action: :call_mcp, + params: %{connection_id: "nonexistent", tool_name: "test", arguments: %{}}, + wait: false, + reasoning: "Testing call_mcp timeout preservation" + } + + test_state = %{ + parent_state + | pending_actions: %{}, + action_counter: 0 + } + + _dispatched = + ActionExecutor.execute_consensus_action(test_state, action_response, self()) + + # call_mcp with a nonexistent connection will fail quickly with an error, + # NOT with :timeout. This proves the 600_000 timeout was applied (not 5000). + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + # Should be an error (no MCP connection) — NOT a timeout + refute match?({:error, :timeout}, result), + "call_mcp should not timeout with 600_000ms. Got: #{inspect(result)}" + after + 10_000 -> + flunk("No action result within 10s for call_mcp") + end + end + end + + # ============================================================================ + # R76: Other actions no timeout override [UNIT] + # + # WHEN ActionExecutor builds execute_opts for :orient + # THEN no timeout key is added (uses default 5000 from always_sync_actions) + # + # Regression test: ensures the case _ clause doesn't leak timeout overrides. + # Orient is in @always_sync_actions so it gets default 5000 from ClientAPI. + # It should complete fast (no external calls) — if it times out, something broke. + # ============================================================================ + + describe "R76: other actions no timeout override" do + @tag :r76 + @tag :unit + test "orient does not get a special timeout override", + %{deps: deps, task: task} do + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("0") + } + + {:ok, parent_pid} = spawn_parent_with_budget(deps, task, parent_budget) + {:ok, parent_state} = Core.get_state(parent_pid) + + action_response = %{ + action: :orient, + params: %{ + current_situation: "Testing timeout override behavior", + goal_clarity: "Verify orient uses default timeout", + available_resources: "Unit test framework", + key_challenges: "None", + delegation_consideration: "Not applicable" + }, + wait: false, + reasoning: "Testing no timeout override for orient" + } + + test_state = %{ + parent_state + | pending_actions: %{}, + action_counter: 0 + } + + _dispatched = + ActionExecutor.execute_consensus_action(test_state, action_response, self()) + + # Orient completes instantly (no external calls), uses default 5000ms timeout. + # Should succeed well within the default timeout. + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + assert match?({:ok, _}, result), + "orient should succeed with default timeout. Got: #{inspect(result)}" + after + 5_000 -> + flunk("No action result within 5s for orient") + end + end + end + + # ============================================================================ + # R77: Parent responsive during adjust_budget [SYSTEM] + # + # WHEN adjust_budget is dispatched via ActionExecutor + # THEN the parent agent remains responsive to other GenServer calls + # + # This passes because Task.Supervisor dispatch is non-blocking — the action + # runs in a background task, not inside Core's GenServer callback. + # Regression test: ensures non-blocking dispatch is preserved. + # ============================================================================ + + describe "R77: parent responsive during adjust_budget" do + @tag :r77 + @tag :system + test "parent agent responsive during adjust_budget execution", + %{deps: deps, task: task} do + parent_budget = %{ + mode: :root, + allocated: Decimal.new("100.00"), + committed: Decimal.new("30.00") + } + + {:ok, parent_pid} = spawn_parent_with_budget(deps, task, parent_budget) + {:ok, parent_state} = Core.get_state(parent_pid) + + child_budget = %{ + mode: :child, + allocated: Decimal.new("30.00"), + committed: Decimal.new("0") + } + + {:ok, _child_pid, child_state} = + spawn_child_under_parent(deps, task, parent_pid, parent_state, child_budget) + + action_response = %{ + action: :adjust_budget, + params: %{child_id: child_state.agent_id, new_budget: "50.00"}, + wait: false, + reasoning: "Testing parent responsiveness" + } + + {:ok, fresh_state} = Core.get_state(parent_pid) + + test_state = %{ + fresh_state + | pending_actions: %{}, + action_counter: 0 + } + + # Dispatch the action (runs in background task) + _dispatched = + ActionExecutor.execute_consensus_action(test_state, action_response, self()) + + # Immediately query parent — should respond because dispatch is non-blocking + assert {:ok, _state} = Core.get_state(parent_pid) + + # Parent is responsive. Now wait for the action to complete. + receive do + {:"$gen_cast", {:action_result, _action_id, result, _opts}} -> + assert match?({:ok, _}, result), + "adjust_budget should complete. Got: #{inspect(result)}" + after + 10_000 -> + flunk("No action result within 10s") + end + end + end +end diff --git a/test/quoracle/agent/consensus_handler_ace_delegate_test.exs b/test/quoracle/agent/consensus_handler_ace_delegate_test.exs index 451acf5..52b5a3d 100644 --- a/test/quoracle/agent/consensus_handler_ace_delegate_test.exs +++ b/test/quoracle/agent/consensus_handler_ace_delegate_test.exs @@ -35,45 +35,6 @@ defmodule Quoracle.Agent.ConsensusHandlerAceDelegateTest do } end - # ========== R34: ACE DELEGATE DEFINED ========== - - describe "R34: inject_ace_context delegate exists" do - test "inject_ace_context/3 callable via ConsensusHandler" do - # Test behavior: call the delegate and verify it returns expected type - state = make_state([], nil) - messages = [make_message("user", "Hello")] - result = ConsensusHandler.inject_ace_context(state, messages, "test-model") - assert is_list(result) - end - - test "format_ace_context/2 callable via ConsensusHandler" do - # Test behavior: call the delegate and verify it returns expected type - result = ConsensusHandler.format_ace_context([], nil) - assert is_binary(result) - end - - test "inject_ace_context accepts state, messages, model_id" do - state = make_state([make_lesson("Test")], nil) - messages = [make_message("user", "Hello")] - model_id = "test-model" - - # Should not raise - function exists with correct arity - result = ConsensusHandler.inject_ace_context(state, messages, model_id) - - assert is_list(result) - end - - test "format_ace_context accepts lessons and model_state" do - lessons = [make_lesson("Test")] - model_state = make_model_state("State") - - # Should not raise - function exists with correct arity - result = ConsensusHandler.format_ace_context(lessons, model_state) - - assert is_binary(result) - end - end - # ========== R35: DELEGATE FORWARDS CORRECTLY ========== describe "R35: delegate forwards to AceInjector" do diff --git a/test/quoracle/agent/consensus_handler_profile_test.exs b/test/quoracle/agent/consensus_handler_profile_test.exs index c1d4888..8692438 100644 --- a/test/quoracle/agent/consensus_handler_profile_test.exs +++ b/test/quoracle/agent/consensus_handler_profile_test.exs @@ -15,7 +15,7 @@ defmodule Quoracle.Agent.ConsensusHandlerProfileTest do SystemPromptInjector.ensure_system_prompts. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.Consensus.SystemPromptInjector diff --git a/test/quoracle/agent/consensus_handler_sent_messages_test.exs b/test/quoracle/agent/consensus_handler_sent_messages_test.exs index 64d99cb..6d3c541 100644 --- a/test/quoracle/agent/consensus_handler_sent_messages_test.exs +++ b/test/quoracle/agent/consensus_handler_sent_messages_test.exs @@ -3,7 +3,7 @@ defmodule Quoracle.Agent.ConsensusHandlerSentMessagesTest do Acceptance test for sent_messages in consensus log broadcasts. Verifies that the actual message content is included, not empty lists. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.ConsensusHandler diff --git a/test/quoracle/agent/consensus_handler_test.exs b/test/quoracle/agent/consensus_handler_test.exs index 54b7eb5..e8b2008 100644 --- a/test/quoracle/agent/consensus_handler_test.exs +++ b/test/quoracle/agent/consensus_handler_test.exs @@ -114,21 +114,6 @@ defmodule Quoracle.Agent.ConsensusHandlerTest do assert_receive :trigger_consensus end - test "handles small integer wait values", %{state: state} do - action = :send_message - wait_seconds = 1 - - result = ConsensusHandler.handle_wait_parameter(state, action, wait_seconds) - - assert %{wait_timer: {timer_ref, :timed_wait}} = result - assert is_reference(timer_ref) - - # Cancel timer and send message immediately for test speed - Process.cancel_timer(timer_ref) - send(self(), :trigger_consensus) - assert_receive :trigger_consensus - end - test "defaults to true for invalid wait values", %{state: state} do action = :call_api @@ -471,66 +456,6 @@ defmodule Quoracle.Agent.ConsensusHandlerTest do result_entry = Enum.find(all_entries, &(&1.type == :result)) assert result_entry != nil, "Result should be stored in history" end - - # R8: Timer Set After Result Cast - test "timed wait sets timer after storing result", %{state: state, agent_pid: agent_pid} do - consensus_result = %{ - action: :orient, - params: %{ - current_situation: "testing", - goal_clarity: "clear", - available_resources: "test suite", - key_challenges: "none", - delegation_consideration: "none" - }, - wait: 5 - } - - # v35.0: Collect async result through helper - result_state = execute_and_collect_result(state, consensus_result, agent_pid) - - histories = result_state.model_histories - all_entries = histories |> Map.values() |> List.flatten() - result_entry = Enum.find(all_entries, &(&1.type == :result)) - assert result_entry != nil, "Result should be stored" - - # Then verify timer was set - assert %{wait_timer: {timer_ref, :timed_wait}} = result_state - assert is_reference(timer_ref) - - # Cleanup - Process.cancel_timer(timer_ref) - end - - # R9: Timed Wait History Alternation (INTEGRATION) - test "timed wait maintains proper message alternation", %{ - state: state, - agent_pid: agent_pid - } do - consensus_result = %{ - action: :orient, - params: %{ - current_situation: "testing timed wait", - goal_clarity: "clear", - available_resources: "test suite", - key_challenges: "none", - delegation_consideration: "none" - }, - wait: 1 - } - - # v35.0: Collect async result through helper - result_state = execute_and_collect_result(state, consensus_result, agent_pid) - - histories = result_state.model_histories - all_entries = histories |> Map.values() |> List.flatten() - result_entry = Enum.find(all_entries, &(&1.type == :result)) - assert result_entry != nil, "Result should be stored" - - # Timer should be set - assert %{wait_timer: {timer_ref, :timed_wait}} = result_state - Process.cancel_timer(timer_ref) - end end describe "missing wait parameter default (R10-R13)" do @@ -557,30 +482,6 @@ defmodule Quoracle.Agent.ConsensusHandlerTest do %{state: state, agent_pid: agent_pid} end - # R10: Default Wait Applied - test "applies default wait: false when nil", %{state: state, agent_pid: agent_pid} do - consensus_result = %{ - action: :orient, - params: %{ - current_situation: "testing", - goal_clarity: "clear", - available_resources: "test suite", - key_challenges: "none", - delegation_consideration: "none" - } - # wait is missing/nil - should default to false, not error - } - - result = ConsensusHandler.execute_consensus_action(state, consensus_result, agent_pid) - - # Should NOT return error - refute match?({:error, :missing_wait_parameter}, result) - - # Should have executed the action and returned state - assert is_map(result) - assert Map.has_key?(result, :agent_id) - end - # R12: Stored Decision Has Wait (INTEGRATION) test "decision stored in history includes defaulted wait: false", %{ state: state, @@ -604,25 +505,6 @@ defmodule Quoracle.Agent.ConsensusHandlerTest do # Consensus triggered via handle_action_result assert_receive :trigger_consensus end - - # R13: No Error on Missing Wait - test "does not return missing_wait_parameter error", %{state: state, agent_pid: agent_pid} do - consensus_result = %{ - action: :spawn_child, - params: %{ - task: "test task" - } - # wait is missing - } - - result = ConsensusHandler.execute_consensus_action(state, consensus_result, agent_pid) - - # Must NOT return error tuple - refute match?({:error, :missing_wait_parameter}, result) - - # Should return state (map) - assert is_map(result) - end end describe "timed wait system behavior (R14)" do diff --git a/test/quoracle/agent/consensus_handler_v13_test.exs b/test/quoracle/agent/consensus_handler_v13_test.exs index 9fb963e..a7d3f8f 100644 --- a/test/quoracle/agent/consensus_handler_v13_test.exs +++ b/test/quoracle/agent/consensus_handler_v13_test.exs @@ -7,7 +7,7 @@ defmodule Quoracle.Agent.ConsensusHandlerV13Test do WorkGroupID: fix-20251211-051748 """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.ConsensusHandler diff --git a/test/quoracle/agent/consensus_pipeline_test.exs b/test/quoracle/agent/consensus_pipeline_test.exs new file mode 100644 index 0000000..022d6bd --- /dev/null +++ b/test/quoracle/agent/consensus_pipeline_test.exs @@ -0,0 +1,353 @@ +defmodule Quoracle.Agent.ConsensusPipelineTest do + @moduledoc """ + Split from ConsensusTest for better parallelism. + Tests refinement process, error handling, configuration, + and JSON response parsing. + """ + + use ExUnit.Case, async: true + + alias Quoracle.Agent.Consensus + + import ExUnit.CaptureLog + import Quoracle.Agent.ConsensusTestHelpers + + describe "refinement process" do + test "triggers refinement when no initial majority" do + prompt = "Ambiguous situation needing refinement" + history = [] + opts = [simulate_no_majority: true, test_mode: true] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, _decision} = result + end + + test "preserves reasoning history during refinement" do + prompt = "Complex decision requiring multiple rounds" + + history = [ + %{role: "user", content: "Previous context"} + ] + + opts = [track_refinement: true, test_mode: true] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, {_type, action, _opts}} = result + assert action.reasoning != "" + end + + test "stops refinement after max rounds" do + prompt = "Never-converging case" + history = [] + opts = [force_max_rounds: true, test_mode: true] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, {type, _action, _opts}} = result + assert type == :forced_decision + end + + test "handles refinement query failures gracefully" do + prompt = "Refinement with potential failures" + history = [] + opts = [simulate_refinement_failure: true, test_mode: true] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, _decision} = result + end + end + + describe "error handling" do + test "returns error when prompt is invalid" do + messages = [] + result = Consensus.get_consensus(messages, test_mode: true) + + assert {:ok, _} = result + end + + test "handles malformed conversation history" do + messages = [ + %{role: "system", content: "System prompt"}, + %{role: "user", content: "Valid message"} + ] + + result = Consensus.get_consensus(messages, test_mode: true) + + assert {:ok, _} = result + end + + test "returns error for nil inputs" do + result = Consensus.get_consensus(nil, test_mode: true) + + assert {:error, reason} = result + assert reason == :invalid_arguments + end + + test "handles partial model failures" do + prompt = "Test with some models failing" + history = [] + opts = [simulate_partial_failure: true, test_mode: true] + + capture_log(fn -> + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, _decision} = result + end) + end + end + + describe "configuration-based test mode" do + test "uses test_mode option instead of Mix.env()" do + prompt = "Test with config" + history = [] + + opts = [test_mode: true] + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + assert {:ok, _decision} = result + end + + test "test mode can be disabled via options" do + prompt = "Production mode test" + history = [] + + opts = [test_mode: true, simulate_failure: false] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + assert {:ok, decision} = result + assert elem(decision, 0) in [:consensus, :forced_decision] + end + + test "checks test_mode option on every call" do + prompt = "Dynamic config test" + history = [] + + opts1 = [test_mode: true] + messages = build_test_messages(prompt, history) + assert {:ok, _result1} = Consensus.get_consensus(messages, opts1) + + opts2 = [test_mode: true, force_no_consensus: true] + messages = build_test_messages(prompt, history) + assert {:ok, result2} = Consensus.get_consensus(messages, opts2) + + assert elem(result2, 0) == :forced_decision + end + end + + describe "JSON response parsing" do + test "parses properly formatted JSON action responses" do + json_response = ~s({ + "action": "spawn_child", + "params": {"task": "analyze data"}, + "reasoning": "Data analysis is needed" + }) + + parsed = Consensus.parse_json_response(json_response) + + assert {:ok, action} = parsed + assert action.action == :spawn_child + assert action.params == %{"task" => "analyze data"} + assert action.reasoning == "Data analysis is needed" + end + + test "handles JSON with extra fields gracefully" do + json_response = ~s({ + "action": "wait", + "params": {"wait": 5000}, + "reasoning": "Need to wait", + "confidence": 0.95, + "model": "gpt-4" + }) + + {:ok, action} = Consensus.parse_json_response(json_response) + + assert action.action == :wait + assert action.params == %{"wait" => 5000} + assert action.reasoning == "Need to wait" + refute Map.has_key?(action, :confidence) + refute Map.has_key?(action, :model) + end + + test "converts string actions to atoms safely" do + json_response = ~s({ + "action": "execute_shell", + "params": {"command": "ls"}, + "reasoning": "List files" + }) + + {:ok, action} = Consensus.parse_json_response(json_response) + + assert action.action == :execute_shell + assert is_atom(action.action) + end + + test "returns error for invalid JSON" do + invalid_json = "not valid json {action:" + + log = + capture_log(fn -> + result = Consensus.parse_json_response(invalid_json) + assert {:error, :invalid_json} = result + end) + + assert log =~ "Failed to" + end + + test "extracts JSON from Markdown code blocks with json tag" do + markdown_wrapped = """ + Here's my response: + + ```json + { + "action": "wait", + "params": {"wait": 1000}, + "reasoning": "Waiting for process to complete" + } + ``` + + That's the action to take. + """ + + {:ok, action} = Consensus.parse_json_response(markdown_wrapped) + + assert action.action == :wait + assert action.params == %{"wait" => 1000} + assert action.reasoning == "Waiting for process to complete" + end + + test "extracts JSON from Markdown code blocks without language tag" do + markdown_wrapped = """ + ``` + { + "action": "orient", + "params": {}, + "reasoning": "Getting my bearings" + } + ``` + """ + + {:ok, action} = Consensus.parse_json_response(markdown_wrapped) + + assert action.action == :orient + assert action.params == %{} + assert action.reasoning == "Getting my bearings" + end + + test "extracts JSON with text before and after" do + wrapped_json = """ + I'll help you with that. Here's my decision: + {"action": "spawn_child", "params": {"task": "subtask"}, "reasoning": "decompose"} + That should work! + """ + + {:ok, action} = Consensus.parse_json_response(wrapped_json) + + assert action.action == :spawn_child + assert action.params == %{"task" => "subtask"} + assert action.reasoning == "decompose" + end + + test "handles complex nested JSON in Markdown" do + complex_wrapped = """ + The response is: + ```json + { + "action": "wait", + "params": { + "nested": { + "value": "test", + "deep": {"level": 3} + } + }, + "reasoning": "complex nesting" + } + ``` + End of response. + """ + + {:ok, action} = Consensus.parse_json_response(complex_wrapped) + + assert action.action == :wait + assert action.params["nested"]["value"] == "test" + assert action.params["nested"]["deep"]["level"] == 3 + assert action.reasoning == "complex nesting" + end + + test "handles JSON with braces in string values" do + json_with_braces = """ + Before text + {"action": "wait", "params": {"msg": "use } and { here"}, "reasoning": "test"} + After text + """ + + {:ok, action} = Consensus.parse_json_response(json_with_braces) + + assert action.action == :wait + assert action.params == %{"msg" => "use } and { here"} + assert action.reasoning == "test" + end + + test "returns error when no JSON found in wrapped text" do + no_json = "This is just plain text with no JSON in it" + + log = + capture_log(fn -> + result = Consensus.parse_json_response(no_json) + assert {:error, :invalid_json} = result + end) + + assert log =~ "Failed to" + end + + test "returns error for missing required fields" do + incomplete_json = ~s({"action": "wait"}) + + result = Consensus.parse_json_response(incomplete_json) + + assert {:error, :missing_fields} = result + end + + test "returns error for unknown action types" do + unknown_action = ~s({ + "action": "unknown_action_xyz", + "params": {}, + "reasoning": "test" + }) + + result = Consensus.parse_json_response(unknown_action) + + assert {:error, :unknown_action} = result + end + + test "handles nested parameter structures" do + json_response = ~s({ + "action": "spawn_child", + "params": { + "task": "complex task", + "config": { + "timeout": 30000, + "retries": 3 + }, + "models": ["gpt-4", "claude"] + }, + "reasoning": "Complex params" + }) + + {:ok, action} = Consensus.parse_json_response(json_response) + + assert action.params["task"] == "complex task" + assert action.params["config"]["timeout"] == 30000 + assert action.params["config"]["retries"] == 3 + assert action.params["models"] == ["gpt-4", "claude"] + end + end +end diff --git a/test/quoracle/agent/consensus_property_integration_test.exs b/test/quoracle/agent/consensus_property_integration_test.exs new file mode 100644 index 0000000..92490eb --- /dev/null +++ b/test/quoracle/agent/consensus_property_integration_test.exs @@ -0,0 +1,130 @@ +defmodule Quoracle.Agent.ConsensusPropertyIntegrationTest do + @moduledoc """ + Split from ConsensusTest for better parallelism. + Tests property-based consensus verification and integration behavior. + Contains the slowest tests (~880ms each for property checks). + """ + + use ExUnit.Case, async: true + use ExUnitProperties + + alias Quoracle.Agent.Consensus + alias Quoracle.Actions.Schema + + import Quoracle.Agent.ConsensusTestHelpers + + describe "property-based tests" do + property "consensus is deterministic with same inputs and seed" do + check all( + prompt <- string(:printable, min_length: 5, max_length: 100), + seed <- integer(1..10000), + max_runs: 5 + ) do + history = [] + opts = [seed: seed, test_mode: true] + + messages = build_test_messages(prompt, history) + result1 = Consensus.get_consensus(messages, opts) + result2 = Consensus.get_consensus(messages, opts) + + assert result1 == result2 + end + end + + property "consensus always returns valid schema-compliant actions" do + check all( + prompt <- string(:printable, min_length: 5, max_length: 100), + include_history <- boolean(), + max_runs: 10 + ) do + history = + if include_history do + [%{role: "user", content: "test context"}] + else + [] + end + + messages = build_test_messages(prompt, history) + + case Consensus.get_consensus(messages, test_mode: true) do + {:ok, {_type, action, _opts}} -> + assert {:ok, _} = Schema.validate_action_type(action.action) + assert is_map(action.params) + assert is_binary(action.reasoning) + + case action.action do + :spawn_child -> + assert Map.has_key?(action.params, :task) + + :send_message -> + assert Map.has_key?(action.params, :to) + assert Map.has_key?(action.params, :content) + + _ -> + assert is_map(action.params) + end + + {:error, reason} -> + assert is_atom(reason) + end + end + end + end + + describe "integration behavior" do + test "always returns a decision - never 'no decision' state" do + scenarios = [ + {"Clear case", [], [test_mode: true]}, + {"Ambiguous case", [], [test_mode: true]}, + {"Complex case", [%{role: "user", content: "context"}], [test_mode: true]}, + {"Critical case", [], [critical: true, test_mode: true]} + ] + + for {prompt, history, opts} <- scenarios do + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + case result do + {:ok, decision} -> + assert elem(decision, 0) in [:consensus, :forced_decision] + {_type, action, _opts} = decision + assert is_map(action) + assert action.action != nil + + {:error, _reason} -> + assert true + + other -> + flunk("Unexpected result: #{inspect(other)}") + end + end + end + + test "decision is deterministic with same inputs" do + prompt = "Deterministic test" + history = [] + opts = [seed: 42, test_mode: true] + + messages = build_test_messages(prompt, history) + result1 = Consensus.get_consensus(messages, opts) + result2 = Consensus.get_consensus(messages, opts) + + assert result1 == result2 + end + + test "uses priority-based tiebreaking for identical vote counts" do + prompt = "Tie scenario" + history = [] + opts = [simulate_tie: true, test_mode: true] + + messages = build_test_messages(prompt, history) + result = Consensus.get_consensus(messages, opts) + + assert {:ok, {_type, action, _opts}} = result + + _conservative_actions = [:orient, :wait, :send_message] + + assert is_atom(action.action) + end + end +end diff --git a/test/quoracle/agent/consensus_retry_test.exs b/test/quoracle/agent/consensus_retry_test.exs index 37c7eb3..adcfbf3 100644 --- a/test/quoracle/agent/consensus_retry_test.exs +++ b/test/quoracle/agent/consensus_retry_test.exs @@ -22,7 +22,7 @@ defmodule Quoracle.Agent.ConsensusRetryTest do - R12: Parent receives notification after exhaustion [SYSTEM] - R73: State field defaults to 0 [UNIT] """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true @moduletag capture_log: true diff --git a/test/quoracle/agent/consensus_test.exs b/test/quoracle/agent/consensus_test.exs index e1aaa03..cc4337a 100644 --- a/test/quoracle/agent/consensus_test.exs +++ b/test/quoracle/agent/consensus_test.exs @@ -6,10 +6,8 @@ defmodule Quoracle.Agent.ConsensusTest do # Tests use async: true with shared mode for Task.async_stream processes use ExUnit.Case, async: true - use ExUnitProperties alias Quoracle.Agent.Consensus - alias Quoracle.Actions.Schema import ExUnit.CaptureLog import Quoracle.Agent.ConsensusTestHelpers @@ -283,507 +281,4 @@ defmodule Quoracle.Agent.ConsensusTest do end end end - - describe "refinement process" do - test "triggers refinement when no initial majority" do - prompt = "Ambiguous situation needing refinement" - history = [] - # Test flag - opts = [simulate_no_majority: true, test_mode: true] - - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - # Should still return a decision after refinement - assert {:ok, _decision} = result - end - - test "preserves reasoning history during refinement" do - prompt = "Complex decision requiring multiple rounds" - - history = [ - %{role: "user", content: "Previous context"} - ] - - # Test flag - opts = [track_refinement: true, test_mode: true] - - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - assert {:ok, {_type, action, _opts}} = result - # Reasoning should reflect refinement process - assert action.reasoning != "" - end - - test "stops refinement after max rounds" do - prompt = "Never-converging case" - history = [] - # Test flag - opts = [force_max_rounds: true, test_mode: true] - - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - assert {:ok, {type, _action, _opts}} = result - # Should force decision after max refinement rounds - assert type == :forced_decision - end - - test "handles refinement query failures gracefully" do - prompt = "Refinement with potential failures" - history = [] - # Test flag - opts = [simulate_refinement_failure: true, test_mode: true] - - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - # Should still return a decision using current round's plurality - assert {:ok, _decision} = result - end - end - - describe "error handling" do - test "returns error when prompt is invalid" do - # With 2-arg signature, validation happens at caller level - # Empty messages array is valid input (system prompt will be injected) - messages = [] - result = Consensus.get_consensus(messages, test_mode: true) - - # Should succeed with injected system prompt (no validation in consensus) - assert {:ok, _} = result - end - - test "handles malformed conversation history" do - # With 2-arg signature, caller must provide valid messages - # Malformed messages will crash at caller level (let it crash) - messages = [ - %{role: "system", content: "System prompt"}, - %{role: "user", content: "Valid message"} - ] - - result = Consensus.get_consensus(messages, test_mode: true) - - # Should succeed with valid messages - assert {:ok, _} = result - end - - test "returns error for nil inputs" do - # With 2-arg signature, nil/invalid inputs return error immediately - result = Consensus.get_consensus(nil, test_mode: true) - - assert {:error, reason} = result - assert reason == :invalid_arguments - end - - test "handles partial model failures" do - prompt = "Test with some models failing" - history = [] - # Test flag - opts = [simulate_partial_failure: true, test_mode: true] - - import ExUnit.CaptureLog - - # Capture logs since partial failures will log errors - capture_log(fn -> - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - # Should still work with remaining models - assert {:ok, _decision} = result - end) - end - end - - describe "configuration-based test mode" do - test "uses test_mode option instead of Mix.env()" do - prompt = "Test with config" - history = [] - - # Should use test mode from options - opts = [test_mode: true] - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - assert {:ok, _decision} = result - end - - test "test mode can be disabled via options" do - prompt = "Production mode test" - history = [] - - # When test_mode is false, consensus will use real ModelQuery - # In test environment without shared mode, parallel tasks crash - # We verify this by checking the simulate_failure flag is NOT used - opts = [test_mode: true, simulate_failure: false] - - # With test_mode true but no simulate_failure, should get normal mock response - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - assert {:ok, decision} = result - assert elem(decision, 0) in [:consensus, :forced_decision] - end - - test "checks test_mode option on every call" do - prompt = "Dynamic config test" - history = [] - - # First call with test mode enabled - should succeed - opts1 = [test_mode: true] - messages = build_test_messages(prompt, history) - assert {:ok, _result1} = Consensus.get_consensus(messages, opts1) - - # Second call also with test mode to verify option is checked each time - # Use different simulation flag to prove it's checked - opts2 = [test_mode: true, force_no_consensus: true] - messages = build_test_messages(prompt, history) - assert {:ok, result2} = Consensus.get_consensus(messages, opts2) - - # Should get forced_decision due to force_no_consensus flag - assert elem(result2, 0) == :forced_decision - end - end - - describe "JSON response parsing" do - test "parses properly formatted JSON action responses" do - json_response = ~s({ - "action": "spawn_child", - "params": {"task": "analyze data"}, - "reasoning": "Data analysis is needed" - }) - - parsed = Consensus.parse_json_response(json_response) - - assert {:ok, action} = parsed - assert action.action == :spawn_child - assert action.params == %{"task" => "analyze data"} - assert action.reasoning == "Data analysis is needed" - end - - test "handles JSON with extra fields gracefully" do - json_response = ~s({ - "action": "wait", - "params": {"wait": 5000}, - "reasoning": "Need to wait", - "confidence": 0.95, - "model": "gpt-4" - }) - - {:ok, action} = Consensus.parse_json_response(json_response) - - # Should ignore extra fields - assert action.action == :wait - assert action.params == %{"wait" => 5000} - assert action.reasoning == "Need to wait" - refute Map.has_key?(action, :confidence) - refute Map.has_key?(action, :model) - end - - test "converts string actions to atoms safely" do - json_response = ~s({ - "action": "execute_shell", - "params": {"command": "ls"}, - "reasoning": "List files" - }) - - {:ok, action} = Consensus.parse_json_response(json_response) - - assert action.action == :execute_shell - assert is_atom(action.action) - end - - test "returns error for invalid JSON" do - invalid_json = "not valid json {action:" - - import ExUnit.CaptureLog - - log = - capture_log(fn -> - result = Consensus.parse_json_response(invalid_json) - assert {:error, :invalid_json} = result - end) - - # Verify it logged the error - assert log =~ "Failed to" - end - - test "extracts JSON from Markdown code blocks with json tag" do - markdown_wrapped = """ - Here's my response: - - ```json - { - "action": "wait", - "params": {"wait": 1000}, - "reasoning": "Waiting for process to complete" - } - ``` - - That's the action to take. - """ - - {:ok, action} = Consensus.parse_json_response(markdown_wrapped) - - assert action.action == :wait - assert action.params == %{"wait" => 1000} - assert action.reasoning == "Waiting for process to complete" - end - - test "extracts JSON from Markdown code blocks without language tag" do - markdown_wrapped = """ - ``` - { - "action": "orient", - "params": {}, - "reasoning": "Getting my bearings" - } - ``` - """ - - {:ok, action} = Consensus.parse_json_response(markdown_wrapped) - - assert action.action == :orient - assert action.params == %{} - assert action.reasoning == "Getting my bearings" - end - - test "extracts JSON with text before and after" do - wrapped_json = """ - I'll help you with that. Here's my decision: - {"action": "spawn_child", "params": {"task": "subtask"}, "reasoning": "decompose"} - That should work! - """ - - {:ok, action} = Consensus.parse_json_response(wrapped_json) - - assert action.action == :spawn_child - assert action.params == %{"task" => "subtask"} - assert action.reasoning == "decompose" - end - - test "handles complex nested JSON in Markdown" do - complex_wrapped = """ - The response is: - ```json - { - "action": "wait", - "params": { - "nested": { - "value": "test", - "deep": {"level": 3} - } - }, - "reasoning": "complex nesting" - } - ``` - End of response. - """ - - {:ok, action} = Consensus.parse_json_response(complex_wrapped) - - assert action.action == :wait - assert action.params["nested"]["value"] == "test" - assert action.params["nested"]["deep"]["level"] == 3 - assert action.reasoning == "complex nesting" - end - - test "handles JSON with braces in string values" do - json_with_braces = """ - Before text - {"action": "wait", "params": {"msg": "use } and { here"}, "reasoning": "test"} - After text - """ - - {:ok, action} = Consensus.parse_json_response(json_with_braces) - - assert action.action == :wait - assert action.params == %{"msg" => "use } and { here"} - assert action.reasoning == "test" - end - - test "returns error when no JSON found in wrapped text" do - no_json = "This is just plain text with no JSON in it" - - import ExUnit.CaptureLog - - log = - capture_log(fn -> - result = Consensus.parse_json_response(no_json) - assert {:error, :invalid_json} = result - end) - - assert log =~ "Failed to" - end - - test "returns error for missing required fields" do - incomplete_json = ~s({"action": "wait"}) - - result = Consensus.parse_json_response(incomplete_json) - - assert {:error, :missing_fields} = result - end - - test "returns error for unknown action types" do - unknown_action = ~s({ - "action": "unknown_action_xyz", - "params": {}, - "reasoning": "test" - }) - - result = Consensus.parse_json_response(unknown_action) - - assert {:error, :unknown_action} = result - end - - test "handles nested parameter structures" do - json_response = ~s({ - "action": "spawn_child", - "params": { - "task": "complex task", - "config": { - "timeout": 30000, - "retries": 3 - }, - "models": ["gpt-4", "claude"] - }, - "reasoning": "Complex params" - }) - - {:ok, action} = Consensus.parse_json_response(json_response) - - assert action.params["task"] == "complex task" - assert action.params["config"]["timeout"] == 30000 - assert action.params["config"]["retries"] == 3 - assert action.params["models"] == ["gpt-4", "claude"] - end - end - - describe "property-based tests" do - property "consensus is deterministic with same inputs and seed" do - check all( - prompt <- string(:printable, min_length: 5, max_length: 100), - seed <- integer(1..10000), - max_runs: 5 - ) do - history = [] - opts = [seed: seed, test_mode: true] - - # Same inputs and seed should produce identical results - messages = build_test_messages(prompt, history) - result1 = Consensus.get_consensus(messages, opts) - result2 = Consensus.get_consensus(messages, opts) - - assert result1 == result2 - end - end - - property "consensus always returns valid schema-compliant actions" do - check all( - prompt <- string(:printable, min_length: 5, max_length: 100), - include_history <- boolean(), - max_runs: 10 - ) do - history = - if include_history do - [%{role: "user", content: "test context"}] - else - [] - end - - messages = build_test_messages(prompt, history) - - case Consensus.get_consensus(messages, test_mode: true) do - {:ok, {_type, action, _opts}} -> - # Action must be valid according to schema - assert {:ok, _} = Schema.validate_action_type(action.action) - assert is_map(action.params) - assert is_binary(action.reasoning) - - # Parameters must match action requirements - case action.action do - :spawn_child -> - assert Map.has_key?(action.params, :task) - - :send_message -> - assert Map.has_key?(action.params, :to) - assert Map.has_key?(action.params, :content) - - _ -> - # Other actions have their own requirements - assert is_map(action.params) - end - - {:error, reason} -> - # Errors are acceptable as long as they're valid error atoms - assert is_atom(reason) - end - end - end - end - - describe "integration behavior" do - test "always returns a decision - never 'no decision' state" do - # Test multiple scenarios - scenarios = [ - {"Clear case", [], [test_mode: true]}, - {"Ambiguous case", [], [test_mode: true]}, - {"Complex case", [%{role: "user", content: "context"}], [test_mode: true]}, - {"Critical case", [], [critical: true, test_mode: true]} - ] - - for {prompt, history, opts} <- scenarios do - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - # Must always return a decision or error, never nil or :no_decision - case result do - {:ok, decision} -> - assert elem(decision, 0) in [:consensus, :forced_decision] - {_type, action, _opts} = decision - assert is_map(action) - assert action.action != nil - - {:error, _reason} -> - # Error is acceptable, as long as it's not nil or :no_decision - assert true - - other -> - flunk("Unexpected result: #{inspect(other)}") - end - end - end - - test "decision is deterministic with same inputs" do - prompt = "Deterministic test" - history = [] - # Test flag for deterministic behavior - opts = [seed: 42, test_mode: true] - - messages = build_test_messages(prompt, history) - result1 = Consensus.get_consensus(messages, opts) - result2 = Consensus.get_consensus(messages, opts) - - assert result1 == result2 - end - - test "uses priority-based tiebreaking for identical vote counts" do - prompt = "Tie scenario" - history = [] - # Test flag - opts = [simulate_tie: true, test_mode: true] - - messages = build_test_messages(prompt, history) - result = Consensus.get_consensus(messages, opts) - - assert {:ok, {_type, action, _opts}} = result - - # Should pick the most conservative action (lowest priority) - # Based on ACTION_Schema priorities - _conservative_actions = [:orient, :wait, :send_message] - - # In a tie, should favor conservative actions - # This depends on the specific tie scenario - assert is_atom(action.action) - end - end end diff --git a/test/quoracle/agent/continuation_context_test.exs b/test/quoracle/agent/continuation_context_test.exs index d514c25..e8787ce 100644 --- a/test/quoracle/agent/continuation_context_test.exs +++ b/test/quoracle/agent/continuation_context_test.exs @@ -3,7 +3,7 @@ defmodule Quoracle.Agent.ContinuationContextTest do Tests for multi-turn conversation context and state utilities. Verifies agents maintain proper conversation history across turns. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.{StateUtils, Consensus} diff --git a/test/quoracle/agent/core_ace_context_test.exs b/test/quoracle/agent/core_ace_context_test.exs index b871766..5c5cd69 100644 --- a/test/quoracle/agent/core_ace_context_test.exs +++ b/test/quoracle/agent/core_ace_context_test.exs @@ -141,7 +141,8 @@ defmodule Quoracle.Agent.Core.ACEContextTest do sandbox_owner: deps.sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, - dynsup: deps.dynsup + dynsup: deps.dynsup, + force_persist: true ] {:ok, {_task, agent_pid}} = create_task_with_cleanup("Test task for ACE", opts) @@ -191,7 +192,8 @@ defmodule Quoracle.Agent.Core.ACEContextTest do sandbox_owner: deps.sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, - dynsup: deps.dynsup + dynsup: deps.dynsup, + force_persist: true ] {:ok, {_task, agent_pid}} = create_task_with_cleanup("Test task for ACE", opts) diff --git a/test/quoracle/agent/core_ace_persistence_test.exs b/test/quoracle/agent/core_ace_persistence_test.exs index bebcdf0..9528c43 100644 --- a/test/quoracle/agent/core_ace_persistence_test.exs +++ b/test/quoracle/agent/core_ace_persistence_test.exs @@ -31,6 +31,7 @@ defmodule Quoracle.Agent.CoreACEPersistenceTest do task_id: task.id, initial_prompt: "Test agent for ACE persistence", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -118,6 +119,7 @@ defmodule Quoracle.Agent.CoreACEPersistenceTest do task_id: task_id, restoration_mode: true, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -156,6 +158,7 @@ defmodule Quoracle.Agent.CoreACEPersistenceTest do agent_id: agent_id, task_id: nil, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -194,6 +197,7 @@ defmodule Quoracle.Agent.CoreACEPersistenceTest do agent_id: agent_id, task_id: fake_task_id, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -241,6 +245,7 @@ defmodule Quoracle.Agent.CoreACEPersistenceTest do task_id: task.id, initial_prompt: "ACE survival test", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } diff --git a/test/quoracle/agent/core_continuation_mcp_test.exs b/test/quoracle/agent/core_continuation_mcp_test.exs new file mode 100644 index 0000000..d4c72f4 --- /dev/null +++ b/test/quoracle/agent/core_continuation_mcp_test.exs @@ -0,0 +1,444 @@ +defmodule Quoracle.Agent.CoreContinuationMcpTest.MockMCPClient do + @moduledoc false + use GenServer + + def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts) + def init(opts), do: {:ok, opts} + + def handle_call({:connect, _config}, _from, state) do + {:reply, {:ok, %{connection_id: "test-conn", tools: []}}, state} + end + + def handle_call(_msg, _from, state), do: {:reply, {:error, :unknown_call}, state} +end + +defmodule Quoracle.Agent.CoreContinuationMcpTest do + @moduledoc """ + Split from CoreTest for better parallelism. + Tests consensus continuation bug fix, MCP client lifecycle, + and dismissing flag for race prevention. + """ + + use Quoracle.DataCase, async: true + + @moduletag capture_log: true + + import ExUnit.CaptureLog + import Test.IsolationHelpers + import Test.AgentTestHelpers + + alias Quoracle.Agent.Core + alias Quoracle.Agent.CoreContinuationMcpTest.MockMCPClient + + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + + parent_pid = self() + initial_prompt = "Hello, I am a test agent" + + {:ok, + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner} + end + + describe "consensus continuation bug fix (WorkGroupID: fix-20250926-203000)" do + @tag :arc_cont_01 + test "handle_info(:trigger_consensus) returns proper GenServer tuple", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + ref = Process.monitor(agent) + + send(agent, :trigger_consensus) + + assert {:ok, _state} = Core.get_state(agent) + + refute_received {:DOWN, ^ref, :process, ^agent, _reason} + Process.demonitor(ref, [:flush]) + end + + @tag :arc_cont_03 + test "consensus continuation succeeds and updates state", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + {:ok, initial_state} = Core.get_state(agent) + initial_counter = initial_state.action_counter + + ref = Process.monitor(agent) + + Core.handle_message(agent, "test message") + + {:ok, new_state} = Core.get_state(agent) + + refute_received {:DOWN, ^ref, :process, ^agent, _reason} + assert new_state.action_counter > initial_counter + Process.demonitor(ref, [:flush]) + end + + @tag :arc_cont_04 + test "consensus continuation failure broadcasts error and continues gracefully", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + ref = Process.monitor(agent) + + capture_log(fn -> + send(agent, :trigger_consensus) + end) + + assert {:ok, _state} = Core.get_state(agent) + + refute_received {:DOWN, ^ref, :process, ^agent, _reason} + Process.demonitor(ref, [:flush]) + end + + @tag :arc_cont_05 + test "action with wait: false triggers automatic consensus continuation", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + capture_log(fn -> + Core.handle_agent_message(agent, "Test message") + end) + + {:ok, state_after_first} = Core.get_state(agent) + first_counter = state_after_first.action_counter + + ref = Process.monitor(agent) + + capture_log(fn -> + Core.handle_message(agent, "continuation message") + end) + + {:ok, state_after_second} = Core.get_state(agent) + + refute_received {:DOWN, ^ref, :process, ^agent, _reason} + assert state_after_second.action_counter > first_counter + Process.demonitor(ref, [:flush]) + end + + @tag :arc_cont_06 + test "timed wait expiration triggers consensus continuation", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + {:ok, initial_state} = Core.get_state(agent) + initial_counter = initial_state.action_counter + + :sys.replace_state(agent, fn state -> + Map.put(state, :wait_timer, {make_ref(), :timed_wait}) + end) + + ref = Process.monitor(agent) + + capture_log(fn -> + send(agent, :trigger_consensus) + end) + + {:ok, new_state} = Core.get_state(agent) + + refute_received {:DOWN, ^ref, :process, ^agent, _reason} + assert new_state.action_counter > initial_counter + Process.demonitor(ref, [:flush]) + end + + @tag :arc_cont_07 + test "multiple sequential actions with wait: false process without crashing", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + ref = Process.monitor(agent) + + capture_log(fn -> + Core.handle_agent_message(agent, "First action") + Core.handle_agent_message(agent, "Second action") + Core.handle_agent_message(agent, "Third action") + end) + + {:ok, state} = Core.get_state(agent) + + refute_receive {:DOWN, ^ref, :process, ^agent, _reason} + + first_history = state.model_histories |> Map.values() |> List.first([]) + assert first_history != [], "Messages should be in history after processing" + Process.demonitor(ref, [:flush]) + end + end + + # ============================================================================ + # MCP Client Lifecycle Tests (v14.0) + # ============================================================================ + describe "MCP client lifecycle" do + setup %{parent_pid: parent_pid, deps: deps, pubsub: pubsub, sandbox_owner: sandbox_owner} do + agent = + start_supervised!( + {Core, + {parent_pid, "MCP test agent", + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + {:ok, agent: agent, pubsub: pubsub} + end + + @tag :arc_mcp_01 + test "R4: mcp_client defaults to nil", %{agent: agent} do + {:ok, state} = Core.get_state(agent) + + assert Map.has_key?(state, :mcp_client) + assert is_nil(state.mcp_client) + end + + @tag :arc_mcp_01 + test "R1: stores MCP client reference in state", %{agent: agent} do + {:ok, mcp_client_pid} = Agent.start_link(fn -> nil end) + + on_exit(fn -> + if Process.alive?(mcp_client_pid), do: Agent.stop(mcp_client_pid, :normal, :infinity) + end) + + GenServer.cast(agent, {:store_mcp_client, mcp_client_pid}) + + {:ok, state} = Core.get_state(agent) + + assert state.mcp_client == mcp_client_pid + end + + @tag :arc_mcp_02 + test "R2: mcp_client passed to Router in opts", %{agent: agent} do + {:ok, mcp_client_pid} = MockMCPClient.start_link() + + on_exit(fn -> + if Process.alive?(mcp_client_pid), do: GenServer.stop(mcp_client_pid, :normal, :infinity) + end) + + GenServer.cast(agent, {:store_mcp_client, mcp_client_pid}) + + {:ok, state} = Core.get_state(agent) + assert state.mcp_client == mcp_client_pid + + action_map = %{action: :call_mcp, params: %{transport: "stdio", command: "test"}} + + result = GenServer.call(agent, {:process_action, action_map, "test-action-id"}) + + refute match?({:error, :mcp_client}, result) + end + + @tag :arc_mcp_03 + test "R3: MCP client stored in agent state for lifecycle management", %{ + parent_pid: parent_pid, + deps: deps, + pubsub: pubsub, + sandbox_owner: sandbox_owner + } do + {:ok, agent} = + Core.start_link( + {parent_pid, "MCP storage test", + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub} + ) + + {:ok, mcp_client} = Agent.start_link(fn -> :running end) + + GenServer.cast(agent, {:store_mcp_client, mcp_client}) + + {:ok, state} = Core.get_state(agent) + assert state.mcp_client == mcp_client + + GenServer.stop(agent, :normal, :infinity) + if Process.alive?(mcp_client), do: Agent.stop(mcp_client) + end + end + + # ============================================================================ + # Dismissing Flag Tests (v19.0) + # ============================================================================ + describe "dismissing flag for race prevention" do + setup %{parent_pid: parent_pid, deps: deps, pubsub: pubsub, sandbox_owner: sandbox_owner} do + agent = + start_supervised!( + {Core, + {parent_pid, "Dismissing flag test agent", + test_mode: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + {:ok, agent: agent} + end + + @tag :arc_dismiss_01 + test "R18: state initializes with dismissing: false", %{agent: agent} do + {:ok, state} = Core.get_state(agent) + + assert Map.has_key?(state, :dismissing) + assert state.dismissing == false + end + + @tag :arc_dismiss_02 + test "R19: set_dismissing/2 updates flag to true", %{agent: agent} do + assert :ok = Core.set_dismissing(agent, true) + + {:ok, state} = Core.get_state(agent) + assert state.dismissing == true + end + + @tag :arc_dismiss_03 + test "R20: set_dismissing/2 updates flag to false", %{agent: agent} do + :ok = Core.set_dismissing(agent, true) + {:ok, state1} = Core.get_state(agent) + assert state1.dismissing == true + + assert :ok = Core.set_dismissing(agent, false) + + {:ok, state2} = Core.get_state(agent) + assert state2.dismissing == false + end + + @tag :arc_dismiss_04 + test "R21: dismissing?/1 returns current flag value", %{agent: agent} do + assert Core.dismissing?(agent) == false + + :ok = Core.set_dismissing(agent, true) + assert Core.dismissing?(agent) == true + + :ok = Core.set_dismissing(agent, false) + assert Core.dismissing?(agent) == false + end + + @tag :arc_dismiss_05 + test "R22: set_dismissing is idempotent", %{agent: agent} do + assert :ok = Core.set_dismissing(agent, true) + assert :ok = Core.set_dismissing(agent, true) + assert :ok = Core.set_dismissing(agent, true) + + {:ok, state1} = Core.get_state(agent) + assert state1.dismissing == true + + assert :ok = Core.set_dismissing(agent, false) + assert :ok = Core.set_dismissing(agent, false) + assert :ok = Core.set_dismissing(agent, false) + + {:ok, state2} = Core.get_state(agent) + assert state2.dismissing == false + end + end +end diff --git a/test/quoracle/agent/core_history_state_test.exs b/test/quoracle/agent/core_history_state_test.exs new file mode 100644 index 0000000..b371fed --- /dev/null +++ b/test/quoracle/agent/core_history_state_test.exs @@ -0,0 +1,434 @@ +defmodule Quoracle.Agent.CoreHistoryStateTest do + @moduledoc """ + Split from CoreTest for better parallelism. + Tests action result processing, wait timers, error handling, + conversation history, and state management. + """ + + use Quoracle.DataCase, async: true + + @moduletag capture_log: true + + import ExUnit.CaptureLog + import Test.IsolationHelpers + import Test.AgentTestHelpers + + alias Quoracle.Agent.Core + + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + + parent_pid = self() + initial_prompt = "Hello, I am a test agent" + + {:ok, + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + pubsub: deps.pubsub, + sandbox_owner: sandbox_owner} + end + + describe "action result processing" do + @tag :arc_func_03 + test "consults LLMs when action completes with result", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + action_id = "action-123" + Core.add_pending_action(agent, action_id, :web_fetch, %{url: "http://example.com"}) + Core.handle_action_result(agent, action_id, {:ok, "Success: fetched data"}) + end + + @tag :arc_func_05 + test "tracks multiple pending async actions correctly", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + Core.add_pending_action(agent, "action-1", :wait, %{wait: 1000}) + Core.add_pending_action(agent, "action-2", :web_fetch, %{url: "http://test.com"}) + Core.add_pending_action(agent, "action-3", :shell, %{command: "echo test"}) + + assert {:ok, pending} = Core.get_pending_actions(agent) + assert Map.has_key?(pending, "action-1") + assert Map.has_key?(pending, "action-2") + assert Map.has_key?(pending, "action-3") + + Core.handle_action_result(agent, "action-2", {:ok, "data"}) + + assert {:ok, pending} = Core.get_pending_actions(agent) + refute Map.has_key?(pending, "action-2") + assert Map.has_key?(pending, "action-1") + assert Map.has_key?(pending, "action-3") + end + + @tag :arc_val_02 + test "validates action IDs match pending actions", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + Core.add_pending_action(agent, "valid-id", :wait, %{}) + + capture_log(fn -> + result = Core.handle_action_result(agent, "invalid-id", {:ok, "data"}) + assert result == :ok + end) + + assert {:ok, pending} = Core.get_pending_actions(agent) + assert Map.has_key?(pending, "valid-id") + end + end + + describe "wait timer behavior" do + @tag :arc_func_04 + test "wait timer only triggers LLM consultation if no other events arrive", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + timer_id = "wait-test-1" + Core.set_wait_timer(agent, 60000, timer_id) + + {:ok, timer_ref} = Core.get_wait_timer(agent) + assert is_reference(timer_ref) + + Process.cancel_timer(timer_ref) + end + + test "only one wait timer active at a time", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + Core.set_wait_timer(agent, 5000, "wait-1") + assert {:ok, timer1} = Core.get_wait_timer(agent) + + Core.set_wait_timer(agent, 3000, "wait-2") + assert {:ok, timer2} = Core.get_wait_timer(agent) + + assert timer1 != timer2 + + Process.cancel_timer(timer2) + end + + test "ignores stale timer messages with generation tracking", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + Core.set_wait_timer(agent, 5000, "wait-1") + Core.set_wait_timer(agent, 3000, "wait-2") + + send(agent, {:wait_timeout, "wait-1", 1}) + + assert {:ok, state} = Core.get_state(agent) + + assert state.wait_timer != nil + {:ok, timer_ref} = Core.get_wait_timer(agent) + assert is_reference(timer_ref) + + Process.cancel_timer(timer_ref) + end + end + + describe "existing error handling" do + @tag :arc_err_01 + test "sends error to parent when consensus fails", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + send(agent, {:agent_error, agent, :consensus_failed}) + + assert_receive {:agent_error, ^agent, :consensus_failed}, 30_000 + end + + @tag :arc_err_02 + test "consults LLMs when action fails semantically", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + action_id = "failing-action" + Core.add_pending_action(agent, action_id, :web_fetch, %{url: "bad-url"}) + Core.handle_action_result(agent, action_id, {:error, :invalid_url}) + end + end + + describe "conversation history" do + test "maintains full conversation history", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + capture_log(fn -> + Core.handle_message(agent, {parent_pid, "Message 1"}) + Core.add_pending_action(agent, "action-1", :execute_shell, %{command: "test"}) + _ = Core.get_state(agent) + Core.handle_action_result(agent, "action-1", {:ok, "Result 1"}) + Core.handle_message(agent, {parent_pid, "Message 2"}) + end) + + assert {:ok, histories} = Core.get_model_histories(agent) + all_entries = histories |> Map.values() |> List.flatten() + + assert length(all_entries) >= 3 + + assert Enum.any?(all_entries, fn h -> + case h.content do + content when is_binary(content) -> content =~ "Message 1" + _ -> false + end + end) + + assert Enum.any?(all_entries, fn h -> + case Map.get(h, :result) do + {:ok, result} when is_binary(result) -> result =~ "Result 1" + _ -> false + end + end) + + assert Enum.any?(all_entries, fn h -> + case h.content do + content when is_binary(content) -> content =~ "Message 2" + _ -> false + end + end) + end + + test "includes timestamps in history entries", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + capture_log(fn -> + Core.handle_message(agent, {parent_pid, "Test message"}) + end) + + assert {:ok, histories} = Core.get_model_histories(agent) + all_entries = histories |> Map.values() |> List.flatten() + + assert Enum.all?(all_entries, fn h -> + Map.has_key?(h, :timestamp) and is_struct(h.timestamp, DateTime) + end) + end + end + + describe "state management" do + test "properly initializes state structure", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + assert {:ok, state} = Core.get_state(agent) + + assert is_binary(state.agent_id) + assert state.parent_pid == parent_pid + assert state.children == [] + assert is_map(state.model_histories) + assert is_map(state.pending_actions) + assert state.wait_timer == nil + assert is_integer(state.action_counter) + end + + test "increments action counter for each action", %{ + parent_pid: parent_pid, + initial_prompt: initial_prompt, + deps: deps, + sandbox_owner: sandbox_owner + } do + agent = + start_supervised!( + {Core, + {parent_pid, initial_prompt, + test_mode: true, + skip_initial_consultation: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: deps.pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + + assert {:ok, state1} = Core.get_state(agent) + _initial_count = state1.action_counter + + :ok = Core.wait_for_ready(agent) + + assert {:ok, state_after_init} = Core.get_state(agent) + counter_after_init = state_after_init.action_counter + + capture_log(fn -> + Core.handle_agent_message(agent, "First action") + Core.handle_agent_message(agent, "Second action") + end) + + assert {:ok, _} = Core.get_state(agent) + assert {:ok, _} = Core.get_state(agent) + + assert {:ok, state2} = Core.get_state(agent) + assert state2.action_counter >= counter_after_init + 1 + end + end +end diff --git a/test/quoracle/agent/core_integration_test.exs b/test/quoracle/agent/core_integration_test.exs index dc699e4..154a3f7 100644 --- a/test/quoracle/agent/core_integration_test.exs +++ b/test/quoracle/agent/core_integration_test.exs @@ -260,9 +260,10 @@ defmodule Quoracle.Agent.CoreIntegrationTest do # Try to kill the task (it may have already completed) Process.exit(dead_task.pid, :kill) - # Task exits with :killed if we killed it, or :normal if it completed first + # Task exits with :killed if we killed it, :normal if it completed first, + # or :noproc if it already exited before Process.monitor was called assert_receive {:DOWN, ^ref, :process, _, reason}, 30_000 - assert reason in [:killed, :normal] + assert reason in [:killed, :normal, :noproc] # Create a live waiter task live_task = diff --git a/test/quoracle/agent/core_persistence_test.exs b/test/quoracle/agent/core_persistence_test.exs index 8eb1c35..c588574 100644 --- a/test/quoracle/agent/core_persistence_test.exs +++ b/test/quoracle/agent/core_persistence_test.exs @@ -49,6 +49,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: nil, initial_prompt: "Test prompt", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -78,7 +79,8 @@ defmodule Quoracle.Agent.CorePersistenceTest do task_id: invalid_task_id, parent_pid: nil, initial_prompt: "Test prompt", - test_mode: true + test_mode: true, + force_persist: true } # Agent should spawn despite persistence failure (defensive) @@ -111,7 +113,8 @@ defmodule Quoracle.Agent.CorePersistenceTest do task_id: task_id, restoration_mode: true, initial_prompt: "Restored prompt", - test_mode: true + test_mode: true, + force_persist: true } # Spawn agent with restoration_mode @@ -138,6 +141,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: nil, initial_prompt: "Parent prompt", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -173,6 +177,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: parent_pid, initial_prompt: "Child prompt", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -217,6 +222,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: fake_parent_pid, initial_prompt: "Orphan child", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -248,6 +254,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do initial_prompt: "Test prompt", models: ["test-model"], test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -380,6 +387,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: nil, initial_prompt: "Parent", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -456,6 +464,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_pid: nil, initial_prompt: "Parent", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -583,6 +592,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_id: "v36-parent", initial_prompt: "Child", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } @@ -629,6 +639,7 @@ defmodule Quoracle.Agent.CorePersistenceTest do parent_id: "v36-parent", initial_prompt: "Orphan child", test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner } diff --git a/test/quoracle/agent/core_test.exs b/test/quoracle/agent/core_test.exs index dbc5fbd..4add4ef 100644 --- a/test/quoracle/agent/core_test.exs +++ b/test/quoracle/agent/core_test.exs @@ -1,17 +1,3 @@ -defmodule Quoracle.Agent.CoreTest.MockMCPClient do - @moduledoc false - use GenServer - - def start_link(opts \\ []), do: GenServer.start_link(__MODULE__, opts) - def init(opts), do: {:ok, opts} - - def handle_call({:connect, _config}, _from, state) do - {:reply, {:ok, %{connection_id: "test-conn", tools: []}}, state} - end - - def handle_call(_msg, _from, state), do: {:reply, {:error, :unknown_call}, state} -end - defmodule Quoracle.Agent.CoreTest do @moduledoc """ Test suite for AGENT_Core - the event-driven GenServer that delegates @@ -29,7 +15,6 @@ defmodule Quoracle.Agent.CoreTest do import Test.AgentTestHelpers alias Quoracle.Agent.Core - alias Quoracle.Agent.CoreTest.MockMCPClient setup %{sandbox_owner: sandbox_owner} do # DataCase already provides sandbox_owner via start_owner! pattern @@ -114,39 +99,6 @@ defmodule Quoracle.Agent.CoreTest do assert Enum.all?(histories, fn {_model, messages} -> messages == [] end) end - @tag :arc_func_02 - test "does not perform initial consultation on startup", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Agent should be ready immediately with no history - {:ok, state} = Core.get_state(agent) - assert state.state == :ready - - # Verify no decisions were made across all model histories - {:ok, histories} = Core.get_model_histories(agent) - all_entries = histories |> Map.values() |> List.flatten() - refute Enum.any?(all_entries, &(&1.type == :decision)) - refute Enum.any?(all_entries, &(&1.type == :prompt)) - end - @tag :arc_func_01 test "generates unique agent_id and registers with Registry", %{ parent_pid: parent_pid, @@ -675,11 +627,6 @@ defmodule Quoracle.Agent.CoreTest do test "handle_internal_message/3 sends internal messages", %{agent: agent} do assert :ok = Core.handle_internal_message(agent, :test_type, %{data: "test"}) end - - test "start_link/3 accepts test_mode option", %{agent: agent} do - # Agent already started in setup with test_mode - assert Process.alive?(agent) - end end describe "message handling" do @@ -785,987 +732,4 @@ defmodule Quoracle.Agent.CoreTest do assert Process.alive?(agent) end end - - describe "action result processing" do - @tag :arc_func_03 - test "consults LLMs when action completes with result", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Add a pending action - function doesn't exist yet - action_id = "action-123" - Core.add_pending_action(agent, action_id, :web_fetch, %{url: "http://example.com"}) - - # Send action result - Core.handle_action_result(agent, action_id, {:ok, "Success: fetched data"}) - - # Should consult LLMs with result (can't verify without implementation) - end - - @tag :arc_func_05 - test "tracks multiple pending async actions correctly", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Add multiple pending actions - Core.add_pending_action(agent, "action-1", :wait, %{wait: 1000}) - Core.add_pending_action(agent, "action-2", :web_fetch, %{url: "http://test.com"}) - Core.add_pending_action(agent, "action-3", :shell, %{command: "echo test"}) - - # Verify all are tracked - assert {:ok, pending} = Core.get_pending_actions(agent) - assert Map.has_key?(pending, "action-1") - assert Map.has_key?(pending, "action-2") - assert Map.has_key?(pending, "action-3") - - # Complete one action - Core.handle_action_result(agent, "action-2", {:ok, "data"}) - - # Verify it's removed from pending - assert {:ok, pending} = Core.get_pending_actions(agent) - refute Map.has_key?(pending, "action-2") - assert Map.has_key?(pending, "action-1") - assert Map.has_key?(pending, "action-3") - end - - @tag :arc_val_02 - test "validates action IDs match pending actions", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Add one pending action - Core.add_pending_action(agent, "valid-id", :wait, %{}) - - # Try to complete non-existent action (captures expected warning) - capture_log(fn -> - result = Core.handle_action_result(agent, "invalid-id", {:ok, "data"}) - - # Should handle gracefully - assert result == :ok - end) - - # Verify the valid action is still pending - assert {:ok, pending} = Core.get_pending_actions(agent) - assert Map.has_key?(pending, "valid-id") - end - end - - describe "wait timer behavior" do - @tag :arc_func_04 - test "wait timer only triggers LLM consultation if no other events arrive", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Set a wait timer (use long duration so it doesn't fire during test) - timer_id = "wait-test-1" - Core.set_wait_timer(agent, 60000, timer_id) - - # Verify timer was set - {:ok, timer_ref} = Core.get_wait_timer(agent) - assert is_reference(timer_ref) - - # Cancel timer before test ends to prevent consensus loop - Process.cancel_timer(timer_ref) - end - - test "only one wait timer active at a time", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Set first timer - Core.set_wait_timer(agent, 5000, "wait-1") - assert {:ok, timer1} = Core.get_wait_timer(agent) - - # Set second timer (should cancel first) - Core.set_wait_timer(agent, 3000, "wait-2") - assert {:ok, timer2} = Core.get_wait_timer(agent) - - # Timers should be different - assert timer1 != timer2 - - # Cancel active timer before test ends to prevent consensus loop - Process.cancel_timer(timer2) - end - - test "ignores stale timer messages with generation tracking", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Set first timer - Core.set_wait_timer(agent, 5000, "wait-1") - - # Set second timer (increments generation) - Core.set_wait_timer(agent, 3000, "wait-2") - - # Manually send stale timer message from first timer with old generation - send(agent, {:wait_timeout, "wait-1", 1}) - - # Send a sync message to ensure the stale timer was processed - assert {:ok, state} = Core.get_state(agent) - - # Verify timer is still set (stale message was ignored) - assert state.wait_timer != nil - {:ok, timer_ref} = Core.get_wait_timer(agent) - assert is_reference(timer_ref) - - # Cancel timer before test ends to prevent consensus loop - Process.cancel_timer(timer_ref) - end - end - - describe "existing error handling" do - @tag :arc_err_01 - test "sends error to parent when consensus fails", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Simulate consensus failure by sending error message - send(agent, {:agent_error, agent, :consensus_failed}) - - # Parent should receive error message if consensus fails - assert_receive {:agent_error, ^agent, :consensus_failed}, 30_000 - end - - @tag :arc_err_02 - test "consults LLMs when action fails semantically", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Add pending action - action_id = "failing-action" - Core.add_pending_action(agent, action_id, :web_fetch, %{url: "bad-url"}) - - # Send semantic error - Core.handle_action_result(agent, action_id, {:error, :invalid_url}) - - # Should consult LLMs about the error (can't verify without implementation) - end - - @tag :arc_err_03 - test "shuts down cleanly when parent process dies", %{ - initial_prompt: initial_prompt, - deps: deps, - sandbox_owner: sandbox_owner - } do - # Start a parent process using Agent (proper OTP process) - parent = start_supervised!({Agent, fn -> :parent_state end}) - - # Start agent with parent pid and test_mode (use isolated registry) - agent = - start_supervised!( - {Core, - {parent, initial_prompt, - test_mode: true, - seed: 42, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Wait for agent to register in Registry (confirms init complete) - # Need to get agent_id first to look up the composite value - agent_id = Core.get_agent_id(agent) - assert [{^agent, composite}] = Registry.lookup(deps.registry, {:agent, agent_id}) - assert composite.parent_pid == parent - - # Monitor agent - ref = Process.monitor(agent) - - # Stop parent (simulates parent death) - Agent.stop(parent, :normal) - - # Agent should detect and shutdown - assert_receive {:DOWN, ^ref, :process, ^agent, reason}, 30_000 - assert reason in [:normal, :noproc, :shutdown] - end - end - - describe "conversation history" do - test "maintains full conversation history", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Send several events - # Capture log to suppress expected validation warnings - capture_log(fn -> - Core.handle_message(agent, {parent_pid, "Message 1"}) - # Add pending action first (required for result to be stored) - Core.add_pending_action(agent, "action-1", :execute_shell, %{command: "test"}) - # Small sync to ensure pending action is registered - _ = Core.get_state(agent) - Core.handle_action_result(agent, "action-1", {:ok, "Result 1"}) - Core.handle_message(agent, {parent_pid, "Message 2"}) - end) - - # Get all model histories - assert {:ok, histories} = Core.get_model_histories(agent) - all_entries = histories |> Map.values() |> List.flatten() - - # Verify history contains all events - assert length(all_entries) >= 3 - - assert Enum.any?(all_entries, fn h -> - case h.content do - content when is_binary(content) -> content =~ "Message 1" - _ -> false - end - end) - - assert Enum.any?(all_entries, fn h -> - # New format: result entries have :result field (content is wrapped JSON string) - case Map.get(h, :result) do - {:ok, result} when is_binary(result) -> result =~ "Result 1" - _ -> false - end - end) - - assert Enum.any?(all_entries, fn h -> - case h.content do - content when is_binary(content) -> content =~ "Message 2" - _ -> false - end - end) - end - - test "includes timestamps in history entries", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Capture log to suppress expected validation warnings - capture_log(fn -> - Core.handle_message(agent, {parent_pid, "Test message"}) - end) - - assert {:ok, histories} = Core.get_model_histories(agent) - all_entries = histories |> Map.values() |> List.flatten() - - # All entries should have timestamps - assert Enum.all?(all_entries, fn h -> - Map.has_key?(h, :timestamp) and is_struct(h.timestamp, DateTime) - end) - end - end - - describe "state management" do - test "properly initializes state structure", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Get state - function doesn't exist yet - assert {:ok, state} = Core.get_state(agent) - - # Verify state structure - assert is_binary(state.agent_id) - assert state.parent_pid == parent_pid - assert state.children == [] - assert is_map(state.model_histories) - assert is_map(state.pending_actions) - assert state.wait_timer == nil - assert is_integer(state.action_counter) - end - - test "increments action counter for each action", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - skip_initial_consultation: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: deps.pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Get initial counter - assert {:ok, state1} = Core.get_state(agent) - _initial_count = state1.action_counter - - # Wait for agent to be ready after initial consultation - :ok = Core.wait_for_ready(agent) - - # Get counter after initial consultation (may have incremented) - assert {:ok, state_after_init} = Core.get_state(agent) - counter_after_init = state_after_init.action_counter - - # Send agent messages that trigger consensus (async cast) - # Capture log to suppress expected validation warnings from mock consensus - capture_log(fn -> - Core.handle_agent_message(agent, "First action") - Core.handle_agent_message(agent, "Second action") - end) - - # v18.0: Deferred consensus batches rapid messages into single consensus cycle. - # First get_state processes casts, second ensures :trigger_consensus runs. - assert {:ok, _} = Core.get_state(agent) - assert {:ok, _} = Core.get_state(agent) - - # Counter should have incremented - with v18.0 batching, two rapid messages - # are batched into ONE consensus cycle, so counter increments by 1 not 2 - assert {:ok, state2} = Core.get_state(agent) - assert state2.action_counter >= counter_after_init + 1 - end - end - - describe "consensus continuation bug fix (WorkGroupID: fix-20250926-203000)" do - @tag :arc_cont_01 - test "handle_info(:trigger_consensus) returns proper GenServer tuple", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Monitor for crashes - ref = Process.monitor(agent) - - # Trigger consensus continuation by sending :trigger_consensus message - send(agent, :trigger_consensus) - - # Verify state is still accessible (proves GenServer didn't crash) - assert {:ok, _state} = Core.get_state(agent) - - # GenServer.call above synchronizes - all prior messages processed - refute_received {:DOWN, ^ref, :process, ^agent, _reason} - Process.demonitor(ref, [:flush]) - end - - # arc_cont_02 removed - now unified with :trigger_consensus (same as arc_cont_01) - - @tag :arc_cont_03 - test "consensus continuation succeeds and updates state", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Get initial action counter - {:ok, initial_state} = Core.get_state(agent) - initial_counter = initial_state.action_counter - - # Monitor for crashes - ref = Process.monitor(agent) - - # Trigger consensus via agent message (properly sets consensus_scheduled: true) - # Direct :trigger_consensus is considered stale without consensus_scheduled flag - Core.handle_message(agent, "test message") - - # GenServer.call synchronizes - all prior messages processed before this returns - {:ok, new_state} = Core.get_state(agent) - - # Now safe to check instantly - sync barrier above guarantees processing complete - refute_received {:DOWN, ^ref, :process, ^agent, _reason} - assert new_state.action_counter > initial_counter - Process.demonitor(ref, [:flush]) - end - - @tag :arc_cont_04 - test "consensus continuation failure broadcasts error and continues gracefully", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Monitor for crashes - ref = Process.monitor(agent) - - # Trigger consensus continuation - even if it succeeds, test that agent doesn't crash - capture_log(fn -> - send(agent, :trigger_consensus) - end) - - # Verify state is accessible (proves no crash) - assert {:ok, _state} = Core.get_state(agent) - - # GenServer.call above synchronizes - all prior messages processed - refute_received {:DOWN, ^ref, :process, ^agent, _reason} - Process.demonitor(ref, [:flush]) - end - - @tag :arc_cont_05 - test "action with wait: false triggers automatic consensus continuation", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Send a message that triggers first consensus - capture_log(fn -> - Core.handle_agent_message(agent, "Test message") - end) - - # Get state after first consensus - {:ok, state_after_first} = Core.get_state(agent) - first_counter = state_after_first.action_counter - - # Monitor for crashes - ref = Process.monitor(agent) - - # Send another message to trigger continuation (properly sets consensus_scheduled) - # Direct :trigger_consensus is stale after first consensus clears the flag - capture_log(fn -> - Core.handle_message(agent, "continuation message") - end) - - # GenServer.call synchronizes - all prior messages processed before this returns - {:ok, state_after_second} = Core.get_state(agent) - - # Now safe to check instantly - sync barrier above guarantees processing complete - refute_received {:DOWN, ^ref, :process, ^agent, _reason} - assert state_after_second.action_counter > first_counter - Process.demonitor(ref, [:flush]) - end - - @tag :arc_cont_06 - test "timed wait expiration triggers consensus continuation", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - # Simulate a timed wait scenario by sending :trigger_consensus - # (This is what would be sent after Process.send_after expires) - {:ok, initial_state} = Core.get_state(agent) - initial_counter = initial_state.action_counter - - # v17.0 TEST-FIX: Set wait_timer to simulate timed wait scenario - # The staleness check requires wait_timer to be set (as it would be - # when a timed wait is active). Without this, the message is correctly - # identified as stale and ignored. - :sys.replace_state(agent, fn state -> - Map.put(state, :wait_timer, {make_ref(), :timed_wait}) - end) - - # Monitor for crashes - ref = Process.monitor(agent) - - capture_log(fn -> - send(agent, :trigger_consensus) - end) - - # GenServer.call synchronizes - all prior messages processed before this returns - {:ok, new_state} = Core.get_state(agent) - - # Now safe to check instantly - sync barrier above guarantees processing complete - refute_received {:DOWN, ^ref, :process, ^agent, _reason} - assert new_state.action_counter > initial_counter - Process.demonitor(ref, [:flush]) - end - - @tag :arc_cont_07 - test "multiple sequential actions with wait: false process without crashing", %{ - parent_pid: parent_pid, - initial_prompt: initial_prompt, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - agent = - start_supervised!( - {Core, - {parent_pid, initial_prompt, - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - # Monitor for crashes - ref = Process.monitor(agent) - - # Trigger multiple messages - with deferred consensus, rapid messages batch together - # The key test is that the agent handles all messages without crashing - capture_log(fn -> - Core.handle_agent_message(agent, "First action") - Core.handle_agent_message(agent, "Second action") - Core.handle_agent_message(agent, "Third action") - end) - - # Get state after processing completes - GenServer.call synchronizes - # (all prior messages in mailbox are processed before this returns) - {:ok, state} = Core.get_state(agent) - - # Verify agent didn't crash during message processing - refute_receive {:DOWN, ^ref, :process, ^agent, _reason} - - # Verify model histories contain message entries (batched or individual) - # This is the key assertion - messages were added to history - first_history = state.model_histories |> Map.values() |> List.first([]) - assert first_history != [], "Messages should be in history after processing" - Process.demonitor(ref, [:flush]) - end - end - - # ============================================================================ - # MCP Client Lifecycle Tests (v14.0) - # WorkGroupID: feat-20251126-023746 - # Packet: 4 (Agent Integration) - # ============================================================================ - describe "MCP client lifecycle" do - setup %{parent_pid: parent_pid, deps: deps, pubsub: pubsub, sandbox_owner: sandbox_owner} do - agent = - start_supervised!( - {Core, - {parent_pid, "MCP test agent", - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - {:ok, agent: agent, pubsub: pubsub} - end - - @tag :arc_mcp_01 - test "R4: mcp_client defaults to nil", %{agent: agent} do - # [UNIT] WHEN agent initialized THEN mcp_client is nil - {:ok, state} = Core.get_state(agent) - - assert Map.has_key?(state, :mcp_client) - assert is_nil(state.mcp_client) - end - - @tag :arc_mcp_01 - test "R1: stores MCP client reference in state", %{agent: agent} do - # [UNIT] WHEN {:store_mcp_client, pid} cast received THEN stores in state.mcp_client - # Create a placeholder process to act as MCP client - # MUST use Agent (not raw spawn) because Core.terminate calls GenServer.stop on mcp_client - {:ok, mcp_client_pid} = Agent.start_link(fn -> nil end) - - # Register cleanup in on_exit to prevent leaks if test fails - on_exit(fn -> - if Process.alive?(mcp_client_pid), do: Agent.stop(mcp_client_pid, :normal, :infinity) - end) - - # Store the MCP client - GenServer.cast(agent, {:store_mcp_client, mcp_client_pid}) - - # Give the cast time to process - {:ok, state} = Core.get_state(agent) - - assert state.mcp_client == mcp_client_pid - end - - @tag :arc_mcp_02 - test "R2: mcp_client passed to Router in opts", %{agent: agent} do - # [INTEGRATION] WHEN action executed IF mcp_client in state THEN passed via opts - # Test: Call :call_mcp action - MCP.execute calls Keyword.fetch!(opts, :mcp_client) - # If mcp_client not in opts, raises KeyError. Test verifies it doesn't crash. - - # Create mock MCP client that handles {:connect, config} calls - # MUST be a GenServer because Core.terminate calls GenServer.stop on mcp_client - {:ok, mcp_client_pid} = MockMCPClient.start_link() - - # Register cleanup in on_exit to prevent leaks if test fails - on_exit(fn -> - if Process.alive?(mcp_client_pid), do: GenServer.stop(mcp_client_pid, :normal, :infinity) - end) - - # Store the MCP client in agent state - GenServer.cast(agent, {:store_mcp_client, mcp_client_pid}) - - # Verify it's stored - {:ok, state} = Core.get_state(agent) - assert state.mcp_client == mcp_client_pid - - # Call MCP action directly via handle_process_action - # This exercises the full opts building path in TestActionHandler - action_map = %{action: :call_mcp, params: %{transport: "stdio", command: "test"}} - - # Execute - if mcp_client not passed in opts, MCP.execute crashes with KeyError - result = GenServer.call(agent, {:process_action, action_map, "test-action-id"}) - - # Should succeed (or fail for MCP-specific reasons, not missing mcp_client) - # KeyError on :mcp_client means opts didn't include it - refute match?({:error, :mcp_client}, result) - end - - @tag :arc_mcp_03 - test "R3: MCP client stored in agent state for lifecycle management", %{ - parent_pid: parent_pid, - deps: deps, - pubsub: pubsub, - sandbox_owner: sandbox_owner - } do - # [UNIT] WHEN mcp_client stored THEN accessible in state for monitoring - # NOTE: MCP client self-terminates via Process.monitor on agent death - # (see MCP.Client.handle_info({:DOWN, ...}) - tested in client_test.exs) - # This test verifies the storage mechanism, not termination. - {:ok, agent} = - Core.start_link( - {parent_pid, "MCP storage test", - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub} - ) - - # Create a mock MCP client GenServer - {:ok, mcp_client} = Agent.start_link(fn -> :running end) - - # Store the MCP client - GenServer.cast(agent, {:store_mcp_client, mcp_client}) - - # Verify it's stored and accessible - {:ok, state} = Core.get_state(agent) - assert state.mcp_client == mcp_client - - # Cleanup - GenServer.stop(agent, :normal, :infinity) - if Process.alive?(mcp_client), do: Agent.stop(mcp_client) - end - end - - # ============================================================================ - # Dismissing Flag Tests (v19.0) - # WorkGroupID: feat-20251224-dismiss-child - # Packet: 1 (Infrastructure) - # ============================================================================ - describe "dismissing flag for race prevention" do - setup %{parent_pid: parent_pid, deps: deps, pubsub: pubsub, sandbox_owner: sandbox_owner} do - agent = - start_supervised!( - {Core, - {parent_pid, "Dismissing flag test agent", - test_mode: true, - sandbox_owner: sandbox_owner, - registry: deps.registry, - dynsup: deps.dynsup, - pubsub: pubsub}}, - shutdown: :infinity - ) - - register_agent_cleanup(agent) - - {:ok, agent: agent} - end - - @tag :arc_dismiss_01 - test "R18: state initializes with dismissing: false", %{agent: agent} do - # [UNIT] WHEN agent initialized THEN state.dismissing is false - {:ok, state} = Core.get_state(agent) - - assert Map.has_key?(state, :dismissing) - assert state.dismissing == false - end - - @tag :arc_dismiss_02 - test "R19: set_dismissing/2 updates flag to true", %{agent: agent} do - # [UNIT] WHEN set_dismissing(pid, true) called THEN state.dismissing becomes true - assert :ok = Core.set_dismissing(agent, true) - - {:ok, state} = Core.get_state(agent) - assert state.dismissing == true - end - - @tag :arc_dismiss_03 - test "R20: set_dismissing/2 updates flag to false", %{agent: agent} do - # [UNIT] WHEN set_dismissing(pid, false) called THEN state.dismissing becomes false - # First set to true - :ok = Core.set_dismissing(agent, true) - {:ok, state1} = Core.get_state(agent) - assert state1.dismissing == true - - # Then set back to false - assert :ok = Core.set_dismissing(agent, false) - - {:ok, state2} = Core.get_state(agent) - assert state2.dismissing == false - end - - @tag :arc_dismiss_04 - test "R21: dismissing?/1 returns current flag value", %{agent: agent} do - # [UNIT] WHEN dismissing?(pid) called THEN returns current dismissing value - # Initial value should be false - assert Core.dismissing?(agent) == false - - # Set to true and check - :ok = Core.set_dismissing(agent, true) - assert Core.dismissing?(agent) == true - - # Set back to false and check - :ok = Core.set_dismissing(agent, false) - assert Core.dismissing?(agent) == false - end - - @tag :arc_dismiss_05 - test "R22: set_dismissing is idempotent", %{agent: agent} do - # [UNIT] WHEN set_dismissing called multiple times with same value THEN no error - # Call multiple times with true - assert :ok = Core.set_dismissing(agent, true) - assert :ok = Core.set_dismissing(agent, true) - assert :ok = Core.set_dismissing(agent, true) - - {:ok, state1} = Core.get_state(agent) - assert state1.dismissing == true - - # Call multiple times with false - assert :ok = Core.set_dismissing(agent, false) - assert :ok = Core.set_dismissing(agent, false) - assert :ok = Core.set_dismissing(agent, false) - - {:ok, state2} = Core.get_state(agent) - assert state2.dismissing == false - end - end end diff --git a/test/quoracle/agent/dyn_sup_enforcement_test.exs b/test/quoracle/agent/dyn_sup_enforcement_test.exs index 704674b..c67be33 100644 --- a/test/quoracle/agent/dyn_sup_enforcement_test.exs +++ b/test/quoracle/agent/dyn_sup_enforcement_test.exs @@ -46,48 +46,5 @@ defmodule Quoracle.Agent.DynSupEnforcementTest do # Call through approved helper should work assert {:ok, _pid} = spawn_agent_with_cleanup(deps.dynsup, config, registry: deps.registry) end - - test "ARC_ENF_03: allows call from IsolationHelpers", %{ - deps: deps, - sandbox_owner: _sandbox_owner - } do - # IsolationHelpers functions are approved callers - # This test itself uses create_isolated_deps which internally may spawn processes - # The fact that setup passed proves IsolationHelpers is approved - assert deps.registry != nil - assert deps.dynsup != nil - end - - test "ARC_ENF_04: error message includes helpful instructions", %{ - deps: deps, - sandbox_owner: sandbox_owner - } do - config = %{ - agent_id: "test-error-message", - registry: deps.registry, - pubsub: deps.pubsub, - sandbox_owner: sandbox_owner - } - - error = - assert_raise RuntimeError, fn -> - DynSup.start_agent(deps.dynsup, config, registry: deps.registry) - end - - # Verify error message includes key information - assert error.message =~ "DynSup.start_agent called directly" - assert error.message =~ "spawn_agent_with_cleanup" - assert error.message =~ "Postgrex" - assert error.message =~ "test/support/agent_test_helpers.ex" - end - end - - describe "production code is allowed" do - test "ARC_ENF_05: enforcement only applies in test environment" do - # This test verifies the Mix.env() == :test check exists - # In production, the enforcement function is never called - # We can't test production behavior from test env, but we document the requirement - assert Mix.env() == :test - end end end diff --git a/test/quoracle/agent/dyn_sup_refactor_test.exs b/test/quoracle/agent/dyn_sup_refactor_test.exs deleted file mode 100644 index 3ee94c9..0000000 --- a/test/quoracle/agent/dyn_sup_refactor_test.exs +++ /dev/null @@ -1,383 +0,0 @@ -defmodule Quoracle.Agent.DynSupRefactorTest do - @moduledoc """ - Tests for AGENT_DynSup refactor to remove named process. - Ensures multiple instances can run concurrently without conflicts. - """ - - use ExUnit.Case, async: true - - alias Quoracle.Agent.DynSup - import Test.AgentTestHelpers - - setup do - # Create isolated PubSub for this test - pubsub_name = :"test_pubsub_#{System.unique_integer([:positive])}" - {:ok, _pubsub} = start_supervised({Phoenix.PubSub, name: pubsub_name}) - - # Create isolated Registry for agent tree cleanup - registry = :"test_registry_#{System.unique_integer([:positive])}" - {:ok, _registry} = start_supervised({Registry, keys: :unique, name: registry}) - - %{pubsub: pubsub_name, registry: registry} - end - - describe "start_link/1 without named process" do - test "starts DynamicSupervisor WITHOUT name registration" do - # Start DynSup with no name option using start_supervised for proper cleanup - pid = start_supervised!({DynSup, []}, shutdown: :infinity) - assert is_pid(pid) - - # Should NOT be registered under module name - # Once refactored, calling with name will fail - # For now, just verify we have a PID-based instance - - # Should be a DynamicSupervisor - {:ok, child_pid} = DynamicSupervisor.start_child(pid, {Agent, fn -> :test end}) - - # Add cleanup for spawned agent - on_exit(fn -> - if Process.alive?(child_pid), do: GenServer.stop(child_pid, :normal, :infinity) - end) - end - - test "allows multiple concurrent instances" do - # Start multiple DynSup instances - assert {:ok, pid1} = start_supervised({DynSup, []}, id: :dynsup1, shutdown: :infinity) - assert {:ok, pid2} = start_supervised({DynSup, []}, id: :dynsup2, shutdown: :infinity) - assert {:ok, pid3} = start_supervised({DynSup, []}, id: :dynsup3, shutdown: :infinity) - - # Add cleanup for DynSup instances - on_exit(fn -> - if Process.alive?(pid1), do: stop_supervised(:dynsup1) - if Process.alive?(pid2), do: stop_supervised(:dynsup2) - if Process.alive?(pid3), do: stop_supervised(:dynsup3) - end) - - # All should be different PIDs - assert pid1 != pid2 - assert pid2 != pid3 - assert pid1 != pid3 - - # Each should be functional independently - agent_spec1 = %{ - id: :agent1, - start: {Agent, :start_link, [fn -> :state1 end]} - } - - agent_spec2 = %{ - id: :agent2, - start: {Agent, :start_link, [fn -> :state2 end]} - } - - agent_spec3 = %{ - id: :agent3, - start: {Agent, :start_link, [fn -> :state3 end]} - } - - assert {:ok, agent_pid1} = DynamicSupervisor.start_child(pid1, agent_spec1) - assert {:ok, agent_pid2} = DynamicSupervisor.start_child(pid2, agent_spec2) - assert {:ok, agent_pid3} = DynamicSupervisor.start_child(pid3, agent_spec3) - - # Add cleanup for spawned agents - on_exit(fn -> - if Process.alive?(agent_pid1), do: GenServer.stop(agent_pid1, :normal, :infinity) - if Process.alive?(agent_pid2), do: GenServer.stop(agent_pid2, :normal, :infinity) - if Process.alive?(agent_pid3), do: GenServer.stop(agent_pid3, :normal, :infinity) - end) - end - - test "does not conflict with other tests running in parallel" do - # This test can run simultaneously with others - assert {:ok, pid} = start_supervised({DynSup, []}, shutdown: :infinity) - - # Add cleanup for DynSup instance - on_exit(fn -> - if Process.alive?(pid), do: stop_supervised(DynSup) - end) - - # Should work without any name conflicts - agent_spec = %{ - id: :test_agent, - start: {Agent, :start_link, [fn -> :parallel_test end]} - } - - assert {:ok, agent_pid} = DynamicSupervisor.start_child(pid, agent_spec) - - # Add cleanup for spawned agent - on_exit(fn -> - if Process.alive?(agent_pid), do: GenServer.stop(agent_pid, :normal, :infinity) - end) - - assert Agent.get(agent_pid, & &1) == :parallel_test - end - end - - describe "get_dynsup_pid/0 PID discovery" do - test "returns PID when DynSup is running under Application supervisor" do - # Simulate application supervisor structure - # Note: In real app, DynSup is started by Quoracle.Supervisor - pid = DynSup.get_dynsup_pid() - - assert is_pid(pid) - - # Should be able to use returned PID for operations - agent_spec = %{ - id: :discovery_test, - start: {Agent, :start_link, [fn -> :found end]} - } - - assert {:ok, _} = DynamicSupervisor.start_child(pid, agent_spec) - end - - test "returns nil when supervisor not found" do - # Test that get_dynsup_pid handles missing supervisor gracefully - # In test context without app supervisor, should return nil - - # The app supervisor is running, so get_dynsup_pid will find it - result = DynSup.get_dynsup_pid() - - # Should return a PID since the supervisor is running - assert is_pid(result) - end - - test "finds DynSup correctly via Supervisor.which_children" do - # This tests the actual lookup mechanism - # Verifies it properly navigates the supervisor hierarchy - - pid = DynSup.get_dynsup_pid() - assert is_pid(pid) - - # The get_dynsup_pid function itself verifies the lookup - # If it returns a PID, it found it in the supervisor tree - end - end - - describe "start_agent/2 with PID-based access" do - setup %{pubsub: pubsub, registry: registry} do - {:ok, dynsup_pid} = start_supervised({DynSup, []}, shutdown: :infinity) - - # Add cleanup for DynSup instance - on_exit(fn -> - if Process.alive?(dynsup_pid), do: stop_supervised(DynSup) - end) - - {:ok, dynsup_pid: dynsup_pid, pubsub: pubsub, registry: registry} - end - - test "starts agent with valid config and dynsup PID", %{ - dynsup_pid: dynsup_pid, - pubsub: pubsub, - registry: registry - } do - config = %{ - agent_id: "test_agent_123", - task: "Test task", - parent_pid: nil, - llm_config: %{model: "test-model", temperature: 0.7}, - pubsub: pubsub - } - - # New signature: start_agent(dynsup_pid, config) - assert {:ok, agent_pid} = - spawn_agent_with_cleanup(dynsup_pid, config, registry: registry) - - assert is_pid(agent_pid) - - # Verify agent is supervised by our DynSup - children = DynamicSupervisor.which_children(dynsup_pid) - assert Enum.any?(children, fn {_, pid, _, _} -> pid == agent_pid end) - end - - test "validates config before starting agent", %{ - dynsup_pid: dynsup_pid, - pubsub: pubsub, - registry: registry - } do - # Missing required fields (agent_id) - invalid_config = %{task: "Test", pubsub: pubsub, registry: registry} - - assert {:error, :invalid_config} = DynSup.start_agent(dynsup_pid, invalid_config) - end - - test "handles agent spawn failures gracefully", %{ - dynsup_pid: dynsup_pid, - pubsub: pubsub, - registry: registry - } do - # DynSup only validates agent_id exists - # Agent.Core will handle invalid config, but won't fail in DynSup - bad_config = - %{ - # Missing agent_id will cause validation error in DynSup - pubsub: pubsub, - registry: registry - } - - assert {:error, :invalid_config} = DynSup.start_agent(dynsup_pid, bad_config) - end - - test "supports transient restart strategy", %{ - dynsup_pid: dynsup_pid, - pubsub: pubsub, - registry: registry - } do - config = %{ - agent_id: "transient_agent", - task: "Test transient", - parent_pid: nil, - llm_config: %{model: "test-model", temperature: 0.5}, - pubsub: pubsub - } - - assert {:ok, agent_pid} = - spawn_agent_with_cleanup(dynsup_pid, config, registry: registry) - - # Terminate agent normally via DynSup helper - should not restart - ref = Process.monitor(agent_pid) - DynSup.terminate_agent(agent_pid) - assert_receive {:DOWN, ^ref, :process, ^agent_pid, reason}, 30_000 - assert reason in [:normal, :noproc, :shutdown] - - children = DynamicSupervisor.which_children(dynsup_pid) - assert Enum.empty?(children) - - # Verify transient restart strategy is configured - # The actual restart behavior on abnormal exit is tested by OTP - # We just verify the configuration is correct - config2 = %{config | agent_id: "transient_agent2"} - - assert {:ok, agent_pid2} = - spawn_agent_with_cleanup(dynsup_pid, config2, registry: registry) - - # Verify the child spec has transient restart - children = DynamicSupervisor.which_children(dynsup_pid) - assert [{_, ^agent_pid2, :worker, _}] = children - end - end - - describe "terminate_agent/1" do - setup %{pubsub: pubsub, registry: registry} do - {:ok, dynsup_pid} = start_supervised({DynSup, []}, shutdown: :infinity) - - # Add cleanup for DynSup instance - on_exit(fn -> - if Process.alive?(dynsup_pid), do: stop_supervised(DynSup) - end) - - config = %{ - agent_id: "term_test", - task: "Test termination", - parent_pid: nil, - llm_config: %{model: "test-model", temperature: 0.5}, - pubsub: pubsub - } - - {:ok, agent_pid} = - spawn_agent_with_cleanup(dynsup_pid, config, registry: registry) - - {:ok, dynsup_pid: dynsup_pid, agent_pid: agent_pid, pubsub: pubsub, registry: registry} - end - - test "terminates agent cleanly", %{dynsup_pid: dynsup_pid, agent_pid: agent_pid} do - # Agent should be running - assert Process.alive?(agent_pid) - - # Monitor before termination - ref = Process.monitor(agent_pid) - - # Terminate it via DynSup helper to ensure proper cleanup - assert :ok = DynSup.terminate_agent(agent_pid) - - # Wait for termination - assert_receive {:DOWN, ^ref, :process, ^agent_pid, _reason}, 30_000 - refute Process.alive?(agent_pid) - - # Should be removed from supervisor - children = DynamicSupervisor.which_children(dynsup_pid) - refute Enum.any?(children, fn {_, pid, _, _} -> pid == agent_pid end) - end - - test "handles termination of non-existent agent", %{} do - # Use Task instead of spawn - task = Task.async(fn -> :ok end) - fake_pid = task.pid - Task.await(task) - - # Should handle gracefully for dead process - assert {:error, :not_found} = DynSup.terminate_agent(fake_pid) - end - end - - # NOTE: Integration tests that use global DynSup moved to - # dyn_sup_integration_test.exs with async: false to prevent - # cross-test contamination when killing global DynSup - - describe "no global state or named resources" do - test "DynSup uses no named ETS tables" do - {:ok, pid} = start_supervised({DynSup, []}, shutdown: :infinity) - - # Add cleanup for DynSup instance - on_exit(fn -> - if Process.alive?(pid), do: stop_supervised(DynSup) - end) - - # Get all ETS tables - all_tables = :ets.all() - - # Filter to named tables (atoms) - named_tables = Enum.filter(all_tables, &is_atom/1) - - # DynSup should not create any named tables - dynsup_tables = - Enum.filter(named_tables, fn name -> - String.contains?(to_string(name), "dynsup") or - String.contains?(to_string(name), "DynSup") - end) - - assert Enum.empty?(dynsup_tables) - end - - test "DynSup registers no global names" do - {:ok, pid} = start_supervised({DynSup, []}, shutdown: :infinity) - - # Add cleanup for DynSup instance - on_exit(fn -> - if Process.alive?(pid), do: stop_supervised(DynSup) - end) - - # DynSup should not be registered by name - # Once refactored, these names won't exist in any registry - - # Check global registry - assert :global.whereis_name(DynSup) == :undefined - assert :global.whereis_name(Quoracle.Agent.DynSup) == :undefined - end - - test "multiple test modules can use DynSup simultaneously" do - # Simulate another test module using DynSup - # Use start_supervised for proper cleanup - pid1 = start_supervised!({DynSup, []}, id: :dynsup_task1, shutdown: :infinity) - pid2 = start_supervised!({DynSup, []}, id: :dynsup_task2, shutdown: :infinity) - - task1 = - Task.async(fn -> - DynamicSupervisor.start_child(pid1, {Agent, fn -> :task1 end}) - end) - - task2 = - Task.async(fn -> - DynamicSupervisor.start_child(pid2, {Agent, fn -> :task2 end}) - end) - - # Both should succeed without conflicts - assert {:ok, agent1} = Task.await(task1) - assert {:ok, agent2} = Task.await(task2) - - # Add cleanup for spawned agents - on_exit(fn -> - if Process.alive?(agent1), do: GenServer.stop(agent1, :normal, :infinity) - if Process.alive?(agent2), do: GenServer.stop(agent2, :normal, :infinity) - end) - end - end -end diff --git a/test/quoracle/agent/dyn_sup_restore_test.exs b/test/quoracle/agent/dyn_sup_restore_test.exs index 98ddae9..b1bad2a 100644 --- a/test/quoracle/agent/dyn_sup_restore_test.exs +++ b/test/quoracle/agent/dyn_sup_restore_test.exs @@ -133,6 +133,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) @@ -336,6 +337,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) @@ -725,7 +727,8 @@ defmodule Quoracle.Agent.DynSupRestoreTest do DynSup.restore_agent(deps.dynsup, db_agent, registry: deps.registry, pubsub: deps.pubsub, - sandbox_owner: sandbox_owner + sandbox_owner: sandbox_owner, + force_persist: true ) assert {:ok, state} = Quoracle.Agent.Core.get_state(restored_pid) @@ -914,6 +917,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) @@ -930,6 +934,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) @@ -1050,6 +1055,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) @@ -1194,6 +1200,7 @@ defmodule Quoracle.Agent.DynSupRestoreTest do pubsub: deps.pubsub, test_mode: true, test_opts: [skip_initial_consultation: true], + force_persist: true, sandbox_owner: sandbox_owner }) diff --git a/test/quoracle/agent/dyn_sup_test.exs b/test/quoracle/agent/dyn_sup_test.exs index ee10e7b..360a3a0 100644 --- a/test/quoracle/agent/dyn_sup_test.exs +++ b/test/quoracle/agent/dyn_sup_test.exs @@ -14,22 +14,6 @@ defmodule Quoracle.Agent.DynSupTest do {:ok, deps: deps} end - describe "start_link/1" do - test "ARC_FUNC_01: DynSup starts and can be found via supervisor", %{deps: deps} do - # Use the isolated DynSup from deps - pid = deps.dynsup - assert is_pid(pid) - assert Process.alive?(pid) - end - - test "allows multiple instances without name conflicts" do - # Since DynSup no longer registers a name, multiple instances can coexist - pid1 = start_supervised!({DynSup, []}, shutdown: :infinity) - pid2 = start_supervised!({DynSup, []}, id: :dynsup2, shutdown: :infinity) - assert pid1 != pid2 - end - end - describe "start_agent/1" do test "ARC_FUNC_02: starts agent with valid config and returns {:ok, pid}", %{deps: deps} do agent_id = "test-agent-#{System.unique_integer([:positive])}" @@ -242,53 +226,6 @@ defmodule Quoracle.Agent.DynSupTest do Registry.lookup(deps.registry, {:agent, "transient-normal"}) == [] end) end - - test "ARC_FUNC_04: DynSup crashes when max restarts exceeded", %{deps: _deps} do - # This test verifies that the supervisor crashes when max restarts are exceeded - # We create an isolated supervisor to avoid affecting other tests - import ExUnit.CaptureLog - - # Use a custom child spec that immediately crashes - child_spec = %{ - id: :crasher, - start: {Task, :start_link, [fn -> raise "intentional crash" end]}, - restart: :permanent, - shutdown: 5000, - type: :worker - } - - # Start an isolated DynamicSupervisor with low restart limits for testing - sup_pid = - start_supervised!( - { - DynamicSupervisor, - # Low limit for faster testing - strategy: :one_for_one, max_restarts: 3, max_seconds: 5 - }, - id: :crasher_sup - ) - - # Monitor the supervisor to detect when it crashes - ref = Process.monitor(sup_pid) - - # Start the crasher child which will immediately crash - # Capture logs to suppress the intentional crash error messages - capture_log(fn -> - {:ok, _child_pid} = DynamicSupervisor.start_child(sup_pid, child_spec) - - # Wait for supervisor to crash after exceeding max restarts - # Need to wait inside capture_log to catch all restart attempts - receive do - {:DOWN, ^ref, :process, ^sup_pid, reason} -> - assert reason in [:shutdown, :noproc, :killed] - after - 2000 -> flunk("Supervisor did not crash after max restarts") - end - end) - - # Verify the supervisor is down - refute Process.alive?(sup_pid) - end end describe "terminate_agent/1" do @@ -325,62 +262,6 @@ defmodule Quoracle.Agent.DynSupTest do end end - describe "supervisor integration" do - test "ARC_ERR_02: DynSup restarts when crashed under Application supervisor" do - # Start a DynamicSupervisor under a test supervisor - # Define a module that implements child_spec/1 for the supervisor - child_spec = %{ - id: :test_dyn_sup, - start: {DynamicSupervisor, :start_link, [[strategy: :one_for_one]]}, - restart: :permanent, - type: :supervisor - } - - # Use start_supervised with a proper supervisor specification - {:ok, sup_pid} = - start_supervised(%{ - id: :test_supervisor, - start: {Supervisor, :start_link, [[child_spec], [strategy: :one_for_one]]}, - type: :supervisor - }) - - # Get DynamicSupervisor PID - [{_, dyn_sup_pid, _, _}] = Supervisor.which_children(sup_pid) - assert Process.alive?(dyn_sup_pid) - - # Monitor and kill DynamicSupervisor - :kill required for abnormal exit - # capture_log suppresses expected termination error - import ExUnit.CaptureLog - ref = Process.monitor(dyn_sup_pid) - - capture_log(fn -> - Process.exit(dyn_sup_pid, :kill) - - # Wait for death confirmation - assert_receive {:DOWN, ^ref, :process, ^dyn_sup_pid, reason}, 30_000 - assert reason in [:killed, :noproc] - end) - - # Wait for supervisor to restart DynamicSupervisor - new_dyn_sup_pid = - Enum.reduce_while(1..50, nil, fn _, _ -> - case Supervisor.which_children(sup_pid) do - [{_, pid, _, _}] when pid != dyn_sup_pid and is_pid(pid) -> - {:halt, pid} - - _ -> - # Brief yield to scheduler instead of sleep - :erlang.yield() - {:cont, nil} - end - end) - - assert new_dyn_sup_pid != nil - assert new_dyn_sup_pid != dyn_sup_pid - assert Process.alive?(new_dyn_sup_pid) - end - end - describe "query functions" do test "list_agents/0 returns all supervised agent PIDs", %{deps: deps} do config1 = %{ diff --git a/test/quoracle/agent/dynsup_pubsub_test.exs b/test/quoracle/agent/dynsup_pubsub_test.exs index 7254629..4eb20bb 100644 --- a/test/quoracle/agent/dynsup_pubsub_test.exs +++ b/test/quoracle/agent/dynsup_pubsub_test.exs @@ -23,40 +23,6 @@ defmodule Quoracle.Agent.DynSupPubSubTest do end describe "start_agent/3 with PubSub injection" do - test "accepts pubsub option and passes to child agent", %{ - dynsup: dynsup, - pubsub: pubsub, - registry: registry, - sandbox_owner: sandbox_owner - } do - agent_id = "test-agent-#{System.unique_integer([:positive])}" - - config = %{ - agent_id: agent_id, - parent_pid: nil, - restart: :temporary, - test_mode: true - } - - # Start agent with explicit pubsub and registry - opts = [pubsub: pubsub, registry: registry, sandbox_owner: sandbox_owner] - {:ok, agent_pid} = DynSup.start_agent(dynsup, config, opts) - - # Wait for initialization and ensure tree cleanup - {:ok, _state} = GenServer.call(agent_pid, :get_state) - - on_exit(fn -> - stop_agent_tree(agent_pid, registry) - end) - - # Agent should have received pubsub in its config - assert Process.alive?(agent_pid) - - # Check that agent registered with pubsub in its state - {:ok, state} = GenServer.call(agent_pid, :get_state) - assert state.pubsub == pubsub - end - test "accepts both registry and pubsub options", %{ dynsup: dynsup, pubsub: pubsub, diff --git a/test/quoracle/agent/event_batching_test.exs b/test/quoracle/agent/event_batching_test.exs index 914d029..c43bba1 100644 --- a/test/quoracle/agent/event_batching_test.exs +++ b/test/quoracle/agent/event_batching_test.exs @@ -25,7 +25,7 @@ defmodule Quoracle.Agent.EventBatchingTest do - P1: Event ordering preserved [PROPERTY] - P2: Flag invariant [PROPERTY] """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true use ExUnitProperties @moduletag capture_log: true diff --git a/test/quoracle/agent/mcp_client_lifecycle_test.exs b/test/quoracle/agent/mcp_client_lifecycle_test.exs new file mode 100644 index 0000000..d754e0e --- /dev/null +++ b/test/quoracle/agent/mcp_client_lifecycle_test.exs @@ -0,0 +1,509 @@ +# A simple GenServer that stands in for an MCP Client. +# Started with GenServer.start (NOT start_link) to avoid linking to the +# test process. This prevents :kill signals from propagating to the test. +defmodule Quoracle.Agent.MCPClientLifecycleTest.MockMCPClient do + @moduledoc false + use GenServer + + def start(_opts \\ []), do: GenServer.start(__MODULE__, :ok) + def init(:ok), do: {:ok, %{}} + def handle_call(:ping, _from, state), do: {:reply, :pong, state} +end + +defmodule Quoracle.Agent.MCPClientLifecycleTest do + @moduledoc """ + Test suite for MCP Client lifecycle monitoring, liveness guards, and transparent recovery. + + WorkGroupID: fix-20260221-mcp-client-lifecycle (Packet 1) + + Covers: + - R1: Core monitors MCP Client on store + - R2: Core clears MCP Client on DOWN + - R3: MCPHelpers re-initializes dead client + - R4: Router retrieval liveness check + - R5: Transparent recovery integration + - R6: Monitor does not affect normal operation + - R7: Multiple deaths recovery + - R8: Race window - DOWN not yet processed + - R9: Core state unaffected by non-MCP DOWN + - R10: MCP client nil after monitor cleanup + - R-ACC: Agent transparent recovery (acceptance) + """ + + use Quoracle.DataCase, async: true + + @moduletag capture_log: true + + import Test.IsolationHelpers + import Test.AgentTestHelpers + + alias Quoracle.Agent.Core + alias Quoracle.Actions.Router.MCPHelpers + alias Quoracle.Agent.MCPClientLifecycleTest.MockMCPClient + + # --------------------------------------------------------------------------- + # Helpers + # --------------------------------------------------------------------------- + + defp spawn_test_agent(context) do + %{sandbox_owner: sandbox_owner, deps: deps, pubsub: pubsub} = context + parent_pid = self() + + agent = + start_supervised!( + {Core, + {parent_pid, "MCP lifecycle test agent", + test_mode: true, + skip_auto_consensus: true, + sandbox_owner: sandbox_owner, + registry: deps.registry, + dynsup: deps.dynsup, + pubsub: pubsub}}, + shutdown: :infinity + ) + + register_agent_cleanup(agent) + {agent, deps} + end + + # Starts a mock MCP client (unlinked) and registers cleanup + defp start_mock_client do + {:ok, pid} = MockMCPClient.start() + + on_exit(fn -> + if Process.alive?(pid), do: GenServer.stop(pid, :normal, :infinity) + end) + + pid + end + + # Stores a mock client in Core and syncs to ensure cast is processed + defp store_mcp_client(agent, client_pid) do + GenServer.cast(agent, {:store_mcp_client, client_pid}) + # Sync - GenServer.call after cast ensures cast is processed first + {:ok, state} = Core.get_state(agent) + assert state.mcp_client == client_pid + :ok + end + + # Kills a process and waits for confirmation of death + defp kill_and_wait(pid) do + ref = Process.monitor(pid) + Process.exit(pid, :shutdown) + + receive do + {:DOWN, ^ref, :process, ^pid, _} -> :ok + after + 5000 -> raise "Timeout waiting for process #{inspect(pid)} to die" + end + end + + # Kills a process and waits for the Core agent to process the :DOWN monitor message. + # kill_and_wait only guarantees the TEST process sees death; the Core GenServer + # may not have processed its own :DOWN yet. This helper polls Core.get_state + # until mcp_client is nil (cleared by Core's :DOWN handler). + defp kill_and_wait_for_core(pid, agent) do + kill_and_wait(pid) + + # Poll until Core has processed the :DOWN and cleared mcp_client + Enum.reduce_while(1..20, nil, fn _, _ -> + {:ok, state} = Core.get_state(agent) + + if state.mcp_client == nil do + {:halt, :ok} + else + # Yield to scheduler so Core can process its :DOWN message + :timer.sleep(1) + {:cont, nil} + end + end) + end + + # --------------------------------------------------------------------------- + # Setup + # --------------------------------------------------------------------------- + + setup %{sandbox_owner: sandbox_owner} do + deps = create_isolated_deps() + + {:ok, deps: deps, pubsub: deps.pubsub, sandbox_owner: sandbox_owner} + end + + # --------------------------------------------------------------------------- + # Unit Tests + # --------------------------------------------------------------------------- + + describe "R1: Core monitors MCP Client on store" do + @tag :arc_mcp_lifecycle + test "storing MCP client PID sets up process monitor", context do + {agent, _deps} = spawn_test_agent(context) + + mock_client = start_mock_client() + + # Store the MCP client in Core + store_mcp_client(agent, mock_client) + + # Kill the mock client - if monitor was set up, Core will receive DOWN + # and clear mcp_client to nil + kill_and_wait(mock_client) + + # Sync with Core to ensure DOWN message (if any) is processed + # After the fix: Core monitors the PID, receives DOWN, clears to nil + # Before the fix: Core does NOT monitor, so mcp_client stays as dead PID + {:ok, state_after} = Core.get_state(agent) + assert state_after.mcp_client == nil + end + end + + describe "R2: Core clears MCP Client on DOWN" do + @tag :arc_mcp_lifecycle + test "MCP client death clears state.mcp_client to nil", context do + {agent, _deps} = spawn_test_agent(context) + + mock_client = start_mock_client() + + # Store and verify + store_mcp_client(agent, mock_client) + + # Kill mock client (triggers DOWN if monitored) + kill_and_wait(mock_client) + + # Sync with Core to ensure it has processed any DOWN message + {:ok, state_after} = Core.get_state(agent) + + # After fix: mcp_client should be nil + # Before fix: mcp_client still holds the dead PID + assert state_after.mcp_client == nil + end + end + + describe "R3: MCPHelpers re-initializes dead client" do + @tag :arc_mcp_lifecycle + test "get_or_init_mcp_client re-initializes when cached PID is dead", context do + {agent, _deps} = spawn_test_agent(context) + + # Start and immediately stop a process to get a dead PID + mock_client = start_mock_client() + GenServer.stop(mock_client, :normal, :infinity) + refute Process.alive?(mock_client) + + # Call MCPHelpers with the dead PID - it should detect it's dead + # and re-initialize (create a new one) + # After fix: `is_nil(mcp_client) or not Process.alive?(mcp_client)` triggers re-init + # Before fix: only `is_nil(mcp_client)` triggers re-init, dead PID returned as-is + result = + MCPHelpers.get_or_init_mcp_client( + mcp_client: mock_client, + agent_pid: agent, + agent_id: Core.get_agent_id(agent), + sandbox_owner: context.sandbox_owner + ) + + # Before fix: result == mock_client (stale dead PID returned unchanged) + # After fix: result is a new alive PID + refute result == mock_client, "Should not return the dead PID" + assert is_pid(result) + assert Process.alive?(result) + + # Cleanup newly created MCP.Client + on_exit(fn -> + if is_pid(result) and Process.alive?(result) do + GenServer.stop(result, :normal, :infinity) + end + end) + end + end + + describe "R4: Router retrieval liveness check" do + @tag :arc_mcp_lifecycle + test "Router treats dead MCP client PID as nil", context do + {agent, _deps} = spawn_test_agent(context) + + # Store a PID that we'll then kill + mock_client = start_mock_client() + store_mcp_client(agent, mock_client) + + # Kill the mock client + kill_and_wait(mock_client) + + # Retrieve via GenServer.call (same path Router uses in execute/3) + # The fix adds a liveness guard in Router.execute/3: + # pid = GenServer.call(agent_pid, :get_mcp_client) + # if pid && Process.alive?(pid), do: pid, else: nil + # + # Before fix: Core returns the dead PID, Router uses it (crash) + # After fix (defense layer 1 - monitoring): Core clears to nil on DOWN + # After fix (defense layer 3 - Router guard): Router checks alive? too + # + # We test that Core has cleared the PID (defense layer 1). + # If Core hasn't processed DOWN yet, the Router guard (layer 3) catches it. + retrieved = GenServer.call(agent, :get_mcp_client) + + # Before fix: retrieved is the dead PID (non-nil) + # After fix: retrieved is nil (Core processed DOWN and cleared state) + assert retrieved == nil + end + end + + describe "R6: Monitor does not affect normal operation" do + @tag :arc_mcp_lifecycle + test "alive MCP client is reused without re-initialization", context do + {agent, _deps} = spawn_test_agent(context) + + mock_client = start_mock_client() + + # Store the MCP client (after fix, this also sets up a monitor) + store_mcp_client(agent, mock_client) + + # Verify the alive client is reused by MCPHelpers + result = + MCPHelpers.get_or_init_mcp_client( + mcp_client: mock_client, + agent_pid: agent, + agent_id: Core.get_agent_id(agent) + ) + + assert result == mock_client + assert Process.alive?(result) + + # Now verify the monitor is actually set up by killing the client + # and checking that Core cleared the state. + # This is the key assertion that fails pre-implementation: + # without Process.monitor in handle_store_mcp_client, Core won't + # receive the DOWN message and won't clear mcp_client. + kill_and_wait(mock_client) + + {:ok, state_after} = Core.get_state(agent) + + assert state_after.mcp_client == nil, + "Monitor should have cleared mcp_client after death (proves monitor was set up during store)" + end + end + + describe "R9: Core state unaffected by non-MCP DOWN" do + @tag :arc_mcp_lifecycle + test "DOWN from unrelated process does not clear mcp_client", context do + {agent, _deps} = spawn_test_agent(context) + + # Store a real MCP client + mock_client = start_mock_client() + store_mcp_client(agent, mock_client) + + # Start an unrelated process, have Core monitor it (simulating some other + # monitored process dying), then verify mcp_client is NOT cleared + {:ok, unrelated} = MockMCPClient.start() + + on_exit(fn -> + if Process.alive?(unrelated), do: GenServer.stop(unrelated, :normal, :infinity) + end) + + kill_and_wait(unrelated) + + # The DOWN from unrelated process should NOT affect mcp_client + {:ok, state_after} = Core.get_state(agent) + assert state_after.mcp_client == mock_client + assert Process.alive?(mock_client) + + # Now verify the monitor IS working by killing the actual MCP client + # This proves selectivity: only MCP client DOWN clears the field + kill_and_wait(mock_client) + + {:ok, state_final} = Core.get_state(agent) + + assert state_final.mcp_client == nil, + "MCP client DOWN should clear mcp_client (proves monitoring is selective)" + end + end + + describe "R10: MCP Client nil after monitor cleanup" do + @tag :arc_mcp_lifecycle + test "get_mcp_client returns nil after MCP client death", context do + {agent, _deps} = spawn_test_agent(context) + + mock_client = start_mock_client() + + # Store and verify via :get_mcp_client call + store_mcp_client(agent, mock_client) + assert GenServer.call(agent, :get_mcp_client) == mock_client + + # Kill and wait for death + kill_and_wait(mock_client) + + # After fix: get_mcp_client returns nil (Core processed DOWN) + # Before fix: get_mcp_client returns the dead PID + result = GenServer.call(agent, :get_mcp_client) + assert result == nil + end + end + + # --------------------------------------------------------------------------- + # Integration Tests + # --------------------------------------------------------------------------- + + describe "R5: Transparent recovery integration" do + @tag :arc_mcp_lifecycle + test "call_mcp action recovers transparently after MCP client death", context do + {agent, _deps} = spawn_test_agent(context) + + # Store a mock MCP client + mock_client = start_mock_client() + store_mcp_client(agent, mock_client) + + # Kill the MCP client + kill_and_wait(mock_client) + + # After the fix, Core should process the DOWN and clear mcp_client + {:ok, state_after_death} = Core.get_state(agent) + assert state_after_death.mcp_client == nil + + # Now MCPHelpers should re-initialize when called with nil + # (This is the transparent recovery path: nil triggers lazy re-init) + new_client = + MCPHelpers.get_or_init_mcp_client( + mcp_client: nil, + agent_pid: agent, + agent_id: Core.get_agent_id(agent), + sandbox_owner: context.sandbox_owner + ) + + assert is_pid(new_client) + assert Process.alive?(new_client) + refute new_client == mock_client + + # Cleanup the new MCP.Client + on_exit(fn -> + if is_pid(new_client) and Process.alive?(new_client) do + GenServer.stop(new_client, :normal, :infinity) + end + end) + end + end + + describe "R7: Multiple deaths recovery" do + @tag :arc_mcp_lifecycle + test "agent recovers from multiple consecutive MCP client deaths", context do + {agent, _deps} = spawn_test_agent(context) + + # Cycle 1: Store, kill, verify recovery + client1 = start_mock_client() + store_mcp_client(agent, client1) + kill_and_wait(client1) + + {:ok, state1} = Core.get_state(agent) + assert state1.mcp_client == nil + + # Cycle 2: Store new client, kill, verify recovery again + client2 = start_mock_client() + store_mcp_client(agent, client2) + kill_and_wait(client2) + + {:ok, state2} = Core.get_state(agent) + assert state2.mcp_client == nil + + # Cycle 3: Store yet another client, verify it's alive and stored + client3 = start_mock_client() + store_mcp_client(agent, client3) + + {:ok, state3} = Core.get_state(agent) + assert state3.mcp_client == client3 + assert Process.alive?(client3) + end + end + + describe "R8: Race window - DOWN not yet processed" do + @tag :arc_mcp_lifecycle + test "Process.alive? guard catches stale PID before DOWN is processed", context do + {agent, _deps} = spawn_test_agent(context) + + mock_client = start_mock_client() + + # Store the MCP client + store_mcp_client(agent, mock_client) + + # Kill the client - don't sync with Core (we want the race window + # where Core hasn't processed DOWN yet but MCPHelpers is called) + kill_and_wait(mock_client) + + # Immediately call MCPHelpers with the dead PID + # Before fix: MCPHelpers sees non-nil PID and returns it (stale) + # After fix: MCPHelpers checks Process.alive? and re-initializes + result = + MCPHelpers.get_or_init_mcp_client( + mcp_client: mock_client, + agent_pid: agent, + agent_id: Core.get_agent_id(agent), + sandbox_owner: context.sandbox_owner + ) + + # The guard should catch the dead PID and trigger re-initialization + refute result == mock_client, "Should not return the stale dead PID" + assert is_pid(result) + assert Process.alive?(result) + + # Cleanup the newly created MCP.Client + on_exit(fn -> + if is_pid(result) and Process.alive?(result) do + GenServer.stop(result, :normal, :infinity) + end + end) + end + end + + # --------------------------------------------------------------------------- + # Acceptance Test + # --------------------------------------------------------------------------- + + describe "R-ACC: Agent transparent recovery" do + @tag :acceptance + @tag :arc_mcp_lifecycle + test "agent recovers from dead MCP client on next call_mcp action", context do + {agent, _deps} = spawn_test_agent(context) + + # 1. Simulate having an MCP client stored in agent state + mock_client = start_mock_client() + store_mcp_client(agent, mock_client) + + # 2. MCP Client dies (simulating crash/timeout/network failure) + # Use kill_and_wait_for_core to ensure Core has processed its :DOWN + # message before asserting (fixes non-deterministic scheduler race) + kill_and_wait_for_core(mock_client, agent) + + # 3. Verify Core has cleared the dead PID (monitor + DOWN processing) + {:ok, state_cleared} = Core.get_state(agent) + + assert state_cleared.mcp_client == nil, + "Core should clear mcp_client to nil after MCP Client death" + + # 4. Verify MCPHelpers would re-initialize on next call_mcp + # (This is the path Router.execute takes for :call_mcp actions) + new_client = + MCPHelpers.get_or_init_mcp_client( + mcp_client: nil, + agent_pid: agent, + agent_id: Core.get_agent_id(agent), + sandbox_owner: context.sandbox_owner + ) + + # 5. POSITIVE ASSERTION: New client is created and alive + assert is_pid(new_client), "MCPHelpers should create a new MCP Client" + assert Process.alive?(new_client), "New MCP Client should be alive" + refute new_client == mock_client, "New client should be different from dead one" + + # 6. NEGATIVE ASSERTION: No stale PID, no permanent failure + # MCPHelpers casts {:store_mcp_client, new_client} internally, + # so after syncing, Core should have the new client + {:ok, _sync} = Core.get_state(agent) + retrieved = GenServer.call(agent, :get_mcp_client) + + refute retrieved == mock_client, + "State should never hold the dead MCP Client PID" + + # Cleanup + on_exit(fn -> + if is_pid(new_client) and Process.alive?(new_client) do + GenServer.stop(new_client, :normal, :infinity) + end + end) + end + end +end diff --git a/test/quoracle/agent/message_flush_test.exs b/test/quoracle/agent/message_flush_test.exs index 4e1d937..197736f 100644 --- a/test/quoracle/agent/message_flush_test.exs +++ b/test/quoracle/agent/message_flush_test.exs @@ -17,7 +17,7 @@ defmodule Quoracle.Agent.MessageFlushTest do - A7: user follow-up message reaches agent promptly [SYSTEM/ACCEPTANCE] - A8: parent message to child not delayed [SYSTEM/ACCEPTANCE] """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true import ExUnit.CaptureLog diff --git a/test/quoracle/agent/reflector_multimodal_test.exs b/test/quoracle/agent/reflector_multimodal_test.exs index ac2cab4..9afbab2 100644 --- a/test/quoracle/agent/reflector_multimodal_test.exs +++ b/test/quoracle/agent/reflector_multimodal_test.exs @@ -13,7 +13,7 @@ defmodule Quoracle.Agent.ReflectorMultimodalTest do stringified content within the reflection messages. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.Reflector alias Quoracle.Utils.ContentStringifier diff --git a/test/quoracle/agent/shell_acked_bug_test.exs b/test/quoracle/agent/shell_acked_bug_test.exs index 880a395..f822b9c 100644 --- a/test/quoracle/agent/shell_acked_bug_test.exs +++ b/test/quoracle/agent/shell_acked_bug_test.exs @@ -8,7 +8,7 @@ defmodule Quoracle.Agent.ShellAckedBugTest do FIX: action_executor.ex async branch now sets acked: true so messages flow during async shell execution. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.MessageHandler diff --git a/test/quoracle/agent/stale_continue_consensus_test.exs b/test/quoracle/agent/stale_continue_consensus_test.exs index 1efbff4..8a219ae 100644 --- a/test/quoracle/agent/stale_continue_consensus_test.exs +++ b/test/quoracle/agent/stale_continue_consensus_test.exs @@ -34,7 +34,7 @@ defmodule Quoracle.Agent.StaleContinueConsensusTest do - A11: Pause Stops Agent After Single Consensus [SYSTEM/ACCEPTANCE] - A12: Rapid External Messages Don't Cause Extra Consensus [SYSTEM/ACCEPTANCE] """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Agent.Core.MessageInfoHandler alias Quoracle.Agent.MessageHandler diff --git a/test/quoracle/agent/state_utils_test.exs b/test/quoracle/agent/state_utils_test.exs index d7cd301..b85a63b 100644 --- a/test/quoracle/agent/state_utils_test.exs +++ b/test/quoracle/agent/state_utils_test.exs @@ -64,30 +64,6 @@ defmodule Quoracle.Agent.StateUtilsTest do assert second_entry.action_type == :call_api end - test "creates timestamped entry with all required fields" do - state = %{model_histories: %{"test-model" => []}} - - updated_state = - StateUtils.add_history_entry_with_action( - state, - :result, - {"action_999", {:error, :timeout}}, - :call_api - ) - - [entry] = first_history(updated_state) - - # Verify all fields present - assert Map.has_key?(entry, :type) - assert Map.has_key?(entry, :content) - assert Map.has_key?(entry, :action_type) - assert Map.has_key?(entry, :timestamp) - - # Verify values - assert entry.type == :result - assert entry.action_type == :call_api - end - test "prepends to history (newest first)" do state = %{ model_histories: %{ diff --git a/test/quoracle/agent/token_manager_test.exs b/test/quoracle/agent/token_manager_test.exs index 6eabe80..d194ed6 100644 --- a/test/quoracle/agent/token_manager_test.exs +++ b/test/quoracle/agent/token_manager_test.exs @@ -106,38 +106,6 @@ defmodule Quoracle.Agent.TokenManagerTest do assert tokens > 0 end - test "estimates tokens for different content types uniformly" do - # tiktoken v5.0: Single encoding for all content types (type option removed) - code_content = """ - def process_data(items) do - items - |> Enum.map(&transform/1) - |> Enum.filter(&valid?/1) - |> Enum.reduce(%{}, &aggregate/2) - end - """ - - # Regular text - text_content = "This is regular text without any special formatting or code" - - # JSON structure - json_content = ~s({"action": "orient", "params": {"key": "value"}}) - - # All content uses cl100k_base encoding uniformly - code_tokens = TokenManager.estimate_tokens(code_content) - text_tokens = TokenManager.estimate_tokens(text_content) - json_tokens = TokenManager.estimate_tokens(json_content) - - assert code_tokens > 0 - assert text_tokens > 0 - assert json_tokens > 0 - - # tiktoken handles all content types accurately - assert is_integer(code_tokens) - assert is_integer(text_tokens) - assert is_integer(json_tokens) - end - test "handles empty or nil content gracefully" do # These functions don't exist yet - will fail assert TokenManager.estimate_tokens("") == 0 diff --git a/test/quoracle/agent/token_manager_tiktoken_test.exs b/test/quoracle/agent/token_manager_tiktoken_test.exs index 801fb99..80bdf20 100644 --- a/test/quoracle/agent/token_manager_tiktoken_test.exs +++ b/test/quoracle/agent/token_manager_tiktoken_test.exs @@ -119,18 +119,6 @@ defmodule Quoracle.Agent.TokenManagerTiktokenTest do end end - describe "R4: nil Input Handling" do - test "returns 0 for nil input" do - assert TokenManager.estimate_tokens(nil) == 0 - end - end - - describe "R5: Empty String Handling" do - test "returns 0 for empty string" do - assert TokenManager.estimate_tokens("") == 0 - end - end - describe "R6: History Token Estimation" do test "estimates history tokens using tiktoken" do # History with file paths should use tiktoken for accurate count @@ -325,14 +313,6 @@ defmodule Quoracle.Agent.TokenManagerTiktokenTest do assert is_integer(tokens) assert tokens >= 0 end - - test "empty history returns 0" do - assert TokenManager.estimate_history_tokens([]) == 0 - end - - test "nil history returns 0" do - assert TokenManager.estimate_history_tokens(nil) == 0 - end end describe "R11: Backward Compatibility - Model Limit" do diff --git a/test/quoracle/boot/agent_revival_test.exs b/test/quoracle/boot/agent_revival_test.exs index c424a02..86c13d4 100644 --- a/test/quoracle/boot/agent_revival_test.exs +++ b/test/quoracle/boot/agent_revival_test.exs @@ -64,7 +64,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop the agent to simulate application restart @@ -119,7 +120,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, {task2, task2_pid}} = @@ -127,7 +129,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop both agents to simulate application restart @@ -192,7 +195,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, {bad_task, bad_task_pid}} = @@ -200,7 +204,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop both agents @@ -254,7 +259,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop the agent @@ -305,7 +311,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop the agent @@ -356,7 +363,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) GenServer.stop(task_pid, :normal, :infinity) @@ -389,7 +397,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) root_agent_id = "root-#{task.id}" @@ -423,7 +432,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, {bad_task, bad_task_pid}} = @@ -431,7 +441,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop both agents @@ -484,7 +495,8 @@ defmodule Quoracle.Boot.AgentRevivalTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop the agent diff --git a/test/quoracle/consensus/batch_async_consensus_test.exs b/test/quoracle/consensus/batch_async_consensus_test.exs index c03e95a..8c2f5ee 100644 --- a/test/quoracle/consensus/batch_async_consensus_test.exs +++ b/test/quoracle/consensus/batch_async_consensus_test.exs @@ -16,7 +16,7 @@ defmodule Quoracle.Consensus.BatchAsyncConsensusTest do - R45: batch_async cluster competes with non-batch clusters - R46: Empty batch_async uses default priority """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true use ExUnitProperties alias Quoracle.Consensus.Aggregator diff --git a/test/quoracle/consensus/batch_consensus_test.exs b/test/quoracle/consensus/batch_consensus_test.exs index a7c17d7..f6ab659 100644 --- a/test/quoracle/consensus/batch_consensus_test.exs +++ b/test/quoracle/consensus/batch_consensus_test.exs @@ -11,7 +11,7 @@ defmodule Quoracle.Consensus.BatchConsensusTest do - R46: Dual key support (string keys from LLM) - R47: format_action_summary for batch_sync """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Consensus.Aggregator alias Quoracle.Consensus.Aggregator.ActionSummary diff --git a/test/quoracle/consensus/manager_test.exs b/test/quoracle/consensus/manager_test.exs index 69d035c..425753c 100644 --- a/test/quoracle/consensus/manager_test.exs +++ b/test/quoracle/consensus/manager_test.exs @@ -6,7 +6,7 @@ defmodule Quoracle.Consensus.ManagerTest do """ # Changed to DataCase for database access (config-driven tests need DB) - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Consensus.Manager # NOTE: Old tests for get_model_pool/0 with hardcoded defaults removed (v3.0) diff --git a/test/quoracle/consensus/prompt_builder_batch_sync_test.exs b/test/quoracle/consensus/prompt_builder_batch_sync_test.exs index 1e474d4..0b07985 100644 --- a/test/quoracle/consensus/prompt_builder_batch_sync_test.exs +++ b/test/quoracle/consensus/prompt_builder_batch_sync_test.exs @@ -9,7 +9,7 @@ defmodule Quoracle.Consensus.PromptBuilderBatchSyncTest do - R80-R82: Usage guidance in guidelines section """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Consensus.PromptBuilder alias Quoracle.Actions.Schema.Metadata diff --git a/test/quoracle/consensus/prompt_builder_docs_test.exs b/test/quoracle/consensus/prompt_builder_docs_test.exs new file mode 100644 index 0000000..bd351fa --- /dev/null +++ b/test/quoracle/consensus/prompt_builder_docs_test.exs @@ -0,0 +1,270 @@ +defmodule Quoracle.Consensus.PromptBuilderDocsTest do + @moduledoc """ + Split from PromptBuilderTest for better parallelism. + Tests action documentation in system prompts: NO_EXECUTE, + call_api, call_mcp, and announcement target docs. + """ + + use ExUnit.Case, async: true + + alias Quoracle.Consensus.PromptBuilder + alias Quoracle.Actions.Schema + + @all_capability_groups [:file_read, :file_write, :external_api, :hierarchy, :local_execution] + + describe "NO_EXECUTE documentation (Packet 2)" do + test "system prompt includes NO_EXECUTE tag documentation" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "NO_EXECUTE" + assert prompt =~ "Prompt Injection Protection" + end + + test "NO_EXECUTE section lists all 5 untrusted actions" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "execute_shell" + assert prompt =~ "fetch_web" + assert prompt =~ "call_api" + assert prompt =~ "call_mcp" + assert prompt =~ "answer_engine" + + assert prompt =~ ~r/(untrusted|wrapped|NO_EXECUTE).*execute_shell/si + end + + test "NO_EXECUTE section lists all 5 trusted actions" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "send_message" + assert prompt =~ "spawn_child" + assert prompt =~ "wait" + assert prompt =~ "orient" + assert prompt =~ "todo" + + assert prompt =~ ~r/(trusted|no wrapping|produce trusted).*send_message/si + end + + test "NO_EXECUTE section includes critical data warning" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "CRITICAL" + + assert prompt =~ ~r/(Content inside NO_EXECUTE.*DATA|NO_EXECUTE.*not instructions)/si + + assert prompt =~ ~r/(CRITICAL|IMPORTANT|WARNING)/i + end + + test "NO_EXECUTE section shows injection attack example" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "NO_EXECUTE_" + + assert prompt =~ ~r/(IGNORE|evil|malicious|attack|injection)/i + + assert prompt =~ ~r/example/i + end + + test "NO_EXECUTE section placement in system prompt" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + actions_header_pos = + case Regex.run(~r/## Available Actions/s, prompt, return: :index) do + [{pos, _} | _] -> pos + _ -> 0 + end + + no_execute_section_pos = + case Regex.run(~r/### CRITICAL: Prompt Injection Protection/si, prompt, return: :index) do + [{pos, _} | _] -> pos + _ -> :not_found + end + + first_schema_pos = + case Regex.run(~r/"type":\s*"object"/s, prompt, return: :index) do + [{pos, _} | _] -> pos + _ -> String.length(prompt) + end + + response_format_pos = + case Regex.run(~r/Response JSON Schema:/s, prompt, return: :index) do + [{pos, _} | _] -> pos + _ -> String.length(prompt) + end + + assert no_execute_section_pos != :not_found + assert no_execute_section_pos > actions_header_pos + + assert no_execute_section_pos < first_schema_pos + + assert no_execute_section_pos < response_format_pos + end + + test "NO_EXECUTE section is comprehensive" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ ~r/(prevent.*injection|security.*boundary|protection)/si + + assert prompt =~ ~r/(analyze|process|treat as data)/si + + assert prompt =~ ~r/(DO NOT.*follow|ignore.*instructions|not.*execute)/si + end + + test "NO_EXECUTE section mentions random ID security" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ ~r/(random|ID|unpredictable)/i + assert prompt =~ ~r/(prevent|stop|block).*injection/si + end + end + + describe "call_api action documentation (Packet 6)" do + test "includes call_api in system prompt actions" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "call_api" + + actions = Schema.list_actions() + assert :call_api in actions + end + + test "includes call_api parameter documentation" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "api_type" + assert prompt =~ "url" + assert prompt =~ "method" + + assert prompt =~ ~r/(call_api.*properties|properties.*call_api)/s + end + + test "includes protocol-specific guidance for REST, GraphQL, and JSON-RPC" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ ~r/(REST.*standard HTTP methods|api_type.*rest)/i + + assert prompt =~ ~r/(GraphQL.*query.*mutation|api_type.*graphql)/i + + assert prompt =~ ~r/(JSON-RPC.*method.*params|api_type.*jsonrpc)/i + end + + test "documents authentication strategies with examples" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ ~r/(bearer.*token|auth_type.*bearer)/i + + assert prompt =~ ~r/(basic.*username.*password|auth_type.*basic)/i + + assert prompt =~ ~r/(oauth.*client|auth_type.*oauth)/i + + assert prompt =~ ~r/(\{\{SECRET:.*\}\}|SECRET:.*token)/i + end + + test "includes call_api usage examples with different protocols" do + examples = PromptBuilder.Sections.build_action_examples() + + assert examples =~ "call_api" + + assert examples =~ "api_type" + end + end + + describe "call_mcp v8.0 schema in system prompt" do + test "R21: call_mcp action appears in system prompt" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "call_mcp" + + json_schema = PromptBuilder.action_to_json_schema(:call_mcp) + assert is_map(json_schema) + assert Map.has_key?(json_schema, "params") + end + + test "R22: call_mcp schema shows 3 distinct modes with isolated parameters" do + json_schema = PromptBuilder.action_to_json_schema(:call_mcp) + + assert Map.has_key?(json_schema["params"], "oneOf"), + "call_mcp should have oneOf structure" + + one_of_variants = json_schema["params"]["oneOf"] + assert is_list(one_of_variants) + + assert length(one_of_variants) == 3, + "call_mcp should have exactly 3 modes (CONNECT, CALL, TERMINATE)" + + param_counts = Enum.map(one_of_variants, &map_size(&1["properties"])) + + assert Enum.all?(param_counts, &(&1 in 2..5)), + "Each mode should have 2-5 params, got: #{inspect(param_counts)}" + + descriptions = Enum.map(one_of_variants, & &1["description"]) + assert Enum.any?(descriptions, &String.contains?(&1, "CONNECT")) + assert Enum.any?(descriptions, &String.contains?(&1, "CALL")) + assert Enum.any?(descriptions, &String.contains?(&1, "TERMINATE")) + + [connect, call_mode, terminate] = one_of_variants + assert "transport" in connect["required"] + assert "connection_id" in call_mode["required"] + assert "tool" in call_mode["required"] + assert "connection_id" in terminate["required"] + assert "terminate" in terminate["required"] + end + + test "R23: call_mcp transport shows stdio and http options" do + json_schema = PromptBuilder.action_to_json_schema(:call_mcp) + + transport_spec = + json_schema["params"]["oneOf"] + |> Enum.find_value(fn variant -> + get_in(variant, ["properties", "transport"]) + end) + + assert transport_spec != nil, "transport parameter should exist in call_mcp schema" + + assert transport_spec["enum"] == ["stdio", "http"] or + transport_spec["enum"] == [:stdio, :http], + "transport should have enum with stdio and http values" + end + end + + describe "announcement target docs (v10.0)" do + test "send_message trusted doc mentions announcement target" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "send_message" + assert prompt =~ "announcement" + end + + test "send_message description documents announcement target" do + description = Schema.get_action_description(:send_message) + + assert is_binary(description) + assert description =~ "announcement" + assert description =~ ~r/(recursive|descendant|subtree|all)/i + end + + test "send_message description distinguishes children from announcement" do + description = Schema.get_action_description(:send_message) + + assert is_binary(description) + assert description =~ "children" + assert description =~ "direct" + end + + test "send_message description notes announcement is one-way" do + description = Schema.get_action_description(:send_message) + + assert is_binary(description) + assert description =~ ~r/(one-way|broadcast|no reply)/i + end + + test "system prompt includes announcement target documentation" do + prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) + + assert prompt =~ "announcement" + + assert prompt =~ ~r/(descendant|recursive|subtree)/i + + assert prompt =~ ~r/children/i + end + end +end diff --git a/test/quoracle/consensus/prompt_builder_enhancement_test.exs b/test/quoracle/consensus/prompt_builder_enhancement_test.exs index f59cd71..447eb9a 100644 --- a/test/quoracle/consensus/prompt_builder_enhancement_test.exs +++ b/test/quoracle/consensus/prompt_builder_enhancement_test.exs @@ -4,7 +4,7 @@ defmodule Quoracle.Consensus.PromptBuilderEnhancementTest do Verifies nested map structures, enum types, and recursive type handling for proper JSON schema generation that guides LLMs. """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Consensus.PromptBuilder alias Quoracle.Actions.Schema diff --git a/test/quoracle/consensus/prompt_builder_test.exs b/test/quoracle/consensus/prompt_builder_test.exs index f2e0343..c42cfaf 100644 --- a/test/quoracle/consensus/prompt_builder_test.exs +++ b/test/quoracle/consensus/prompt_builder_test.exs @@ -1,5 +1,5 @@ defmodule Quoracle.Consensus.PromptBuilderTest do - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true use ExUnitProperties alias Quoracle.Consensus.PromptBuilder @@ -16,7 +16,6 @@ defmodule Quoracle.Consensus.PromptBuilderTest do # Verify all actions from Schema are present actions = Schema.list_actions() - assert length(actions) == 22 Enum.each(actions, fn action -> action_string = Atom.to_string(action) @@ -577,338 +576,4 @@ defmodule Quoracle.Consensus.PromptBuilderTest do assert normalize.(prompt1) == normalize.(prompt2) end end - - describe "NO_EXECUTE documentation (Packet 2)" do - # R12: NO_EXECUTE Section Presence - test "system prompt includes NO_EXECUTE tag documentation" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should contain NO_EXECUTE section - assert prompt =~ "NO_EXECUTE" - assert prompt =~ "Prompt Injection Protection" - end - - # R13: Untrusted Actions Listed - test "NO_EXECUTE section lists all 5 untrusted actions" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # All 5 untrusted actions should be listed - assert prompt =~ "execute_shell" - assert prompt =~ "fetch_web" - assert prompt =~ "call_api" - assert prompt =~ "call_mcp" - assert prompt =~ "answer_engine" - - # Should be in context of untrusted/wrapped actions - assert prompt =~ ~r/(untrusted|wrapped|NO_EXECUTE).*execute_shell/si - end - - # R14: Trusted Actions Listed - test "NO_EXECUTE section lists all 5 trusted actions" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # All 5 trusted actions should be listed - assert prompt =~ "send_message" - assert prompt =~ "spawn_child" - assert prompt =~ "wait" - assert prompt =~ "orient" - assert prompt =~ "todo" - - # Should be in context of trusted/not wrapped actions - assert prompt =~ ~r/(trusted|no wrapping|produce trusted).*send_message/si - end - - # R15: Critical Warning Present - test "NO_EXECUTE section includes critical data warning" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should include critical warning about treating content as data - assert prompt =~ "CRITICAL" - - assert prompt =~ ~r/(Content inside NO_EXECUTE.*DATA|NO_EXECUTE.*not instructions)/si - - # Warning should be emphatic - assert prompt =~ ~r/(CRITICAL|IMPORTANT|WARNING)/i - end - - # R16: Example Injection Shown - test "NO_EXECUTE section shows injection attack example" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should include example of injection attempt - assert prompt =~ "NO_EXECUTE_" - - # Example should show malicious content being wrapped - assert prompt =~ ~r/(IGNORE|evil|malicious|attack|injection)/i - - # Should demonstrate the protection mechanism - assert prompt =~ ~r/example/i - end - - # R17: Section Placement - NO_EXECUTE primes security awareness BEFORE action schemas - test "NO_EXECUTE section placement in system prompt" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Find positions of key sections - actions_header_pos = - case Regex.run(~r/## Available Actions/s, prompt, return: :index) do - [{pos, _} | _] -> pos - _ -> 0 - end - - # Match the specific section heading (now a subsection under Available Actions) - no_execute_section_pos = - case Regex.run(~r/### CRITICAL: Prompt Injection Protection/si, prompt, return: :index) do - [{pos, _} | _] -> pos - _ -> :not_found - end - - # Find first action schema (after the NO_EXECUTE warning) - first_schema_pos = - case Regex.run(~r/"type":\s*"object"/s, prompt, return: :index) do - [{pos, _} | _] -> pos - _ -> String.length(prompt) - end - - # Use more specific regex to match the actual "Response JSON Schema:" header - response_format_pos = - case Regex.run(~r/Response JSON Schema:/s, prompt, return: :index) do - [{pos, _} | _] -> pos - _ -> String.length(prompt) - end - - # NO_EXECUTE section should exist and appear after Available Actions header - assert no_execute_section_pos != :not_found - assert no_execute_section_pos > actions_header_pos - - # NO_EXECUTE section should appear BEFORE action schemas (security priming) - assert no_execute_section_pos < first_schema_pos - - # NO_EXECUTE section should appear before response format instructions - assert no_execute_section_pos < response_format_pos - end - - test "NO_EXECUTE section is comprehensive" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should explain the purpose - assert prompt =~ ~r/(prevent.*injection|security.*boundary|protection)/si - - # Should explain what to do with wrapped content - assert prompt =~ ~r/(analyze|process|treat as data)/si - - # Should explain what NOT to do - assert prompt =~ ~r/(DO NOT.*follow|ignore.*instructions|not.*execute)/si - end - - test "NO_EXECUTE section mentions random ID security" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should explain that random ID prevents simple injection - assert prompt =~ ~r/(random|ID|unpredictable)/i - assert prompt =~ ~r/(prevent|stop|block).*injection/si - end - end - - describe "call_api action documentation (Packet 6)" do - # R1: Schema Documentation - test "includes call_api in system prompt actions" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # call_api should be listed as an available action - assert prompt =~ "call_api" - - # Verify it's in the actions list from Schema - actions = Schema.list_actions() - assert :call_api in actions - end - - # R2: Parameter Documentation - test "includes call_api parameter documentation" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should document key call_api parameters in JSON schema - assert prompt =~ "api_type" - assert prompt =~ "url" - assert prompt =~ "method" - - # Should have JSON schema properties for call_api - assert prompt =~ ~r/(call_api.*properties|properties.*call_api)/s - end - - # R3: Protocol Guidance - test "includes protocol-specific guidance for REST, GraphQL, and JSON-RPC" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should have explicit REST guidance - assert prompt =~ ~r/(REST.*standard HTTP methods|api_type.*rest)/i - - # Should have explicit GraphQL guidance - assert prompt =~ ~r/(GraphQL.*query.*mutation|api_type.*graphql)/i - - # Should have explicit JSON-RPC guidance - assert prompt =~ ~r/(JSON-RPC.*method.*params|api_type.*jsonrpc)/i - end - - # R4: Authentication Guidance - test "documents authentication strategies with examples" do - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # Should have example showing bearer token authentication - assert prompt =~ ~r/(bearer.*token|auth_type.*bearer)/i - - # Should have example showing basic authentication - assert prompt =~ ~r/(basic.*username.*password|auth_type.*basic)/i - - # Should have example showing OAuth2 - assert prompt =~ ~r/(oauth.*client|auth_type.*oauth)/i - - # Should show secret resolution pattern for API keys - assert prompt =~ ~r/(\{\{SECRET:.*\}\}|SECRET:.*token)/i - end - - test "includes call_api usage examples with different protocols" do - examples = PromptBuilder.Sections.build_action_examples() - - # Should have REST example - assert examples =~ "call_api" - - # Example should show api_type - assert examples =~ "api_type" - end - end - - # CONSENSUS_PromptBuilder v8.0 - MCP Action Schema - # ARC Verification Criteria for call_mcp schema propagation - describe "call_mcp v8.0 schema in system prompt" do - test "R21: call_mcp action appears in system prompt" do - # [UNIT] - WHEN system prompt built THEN call_mcp action schema included - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # call_mcp should be present in the prompt - assert prompt =~ "call_mcp" - - # Should have its schema documented - json_schema = PromptBuilder.action_to_json_schema(:call_mcp) - assert is_map(json_schema) - assert Map.has_key?(json_schema, "params") - end - - test "R22: call_mcp schema shows 3 distinct modes with isolated parameters" do - # [UNIT] - WHEN call_mcp schema generated THEN shows 3 oneOf modes: CONNECT, CALL, TERMINATE - json_schema = PromptBuilder.action_to_json_schema(:call_mcp) - - # Should have oneOf structure - assert Map.has_key?(json_schema["params"], "oneOf"), - "call_mcp should have oneOf structure" - - one_of_variants = json_schema["params"]["oneOf"] - assert is_list(one_of_variants) - - # Must have exactly 3 modes - assert length(one_of_variants) == 3, - "call_mcp should have exactly 3 modes (CONNECT, CALL, TERMINATE)" - - # Each mode should have only its relevant parameters (2-5 params), not all 9 - param_counts = Enum.map(one_of_variants, &map_size(&1["properties"])) - - assert Enum.all?(param_counts, &(&1 in 2..5)), - "Each mode should have 2-5 params, got: #{inspect(param_counts)}" - - # Verify mode descriptions - descriptions = Enum.map(one_of_variants, & &1["description"]) - assert Enum.any?(descriptions, &String.contains?(&1, "CONNECT")) - assert Enum.any?(descriptions, &String.contains?(&1, "CALL")) - assert Enum.any?(descriptions, &String.contains?(&1, "TERMINATE")) - - # Verify required fields per mode - [connect, call_mode, terminate] = one_of_variants - assert "transport" in connect["required"] - assert "connection_id" in call_mode["required"] - assert "tool" in call_mode["required"] - assert "connection_id" in terminate["required"] - assert "terminate" in terminate["required"] - end - - test "R23: call_mcp transport shows stdio and http options" do - # [UNIT] - WHEN call_mcp schema generated THEN transport shows enum values - json_schema = PromptBuilder.action_to_json_schema(:call_mcp) - - # Find transport parameter in the oneOf variants - transport_spec = - json_schema["params"]["oneOf"] - |> Enum.find_value(fn variant -> - get_in(variant, ["properties", "transport"]) - end) - - assert transport_spec != nil, "transport parameter should exist in call_mcp schema" - - # Transport should be an enum with stdio and http values - assert transport_spec["enum"] == ["stdio", "http"] or - transport_spec["enum"] == [:stdio, :http], - "transport should have enum with stdio and http values" - end - end - - # CONSENSUS_PromptBuilder v10.0 - Announcement Target Documentation - # ARC Verification Criteria for announcement target in system prompts - describe "announcement target docs (v10.0)" do - # R1: Announcement in Trusted Action Docs - test "send_message trusted doc mentions announcement target" do - # [UNIT] - WHEN prepare_action_docs called THEN send_message entry mentions announcement target - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # The send_message documentation should mention announcement as a target option - # Both concepts must be present in the prompt - assert prompt =~ "send_message" - assert prompt =~ "announcement" - end - - # R2: Action Description Updated - test "send_message description documents announcement target" do - # [UNIT] - WHEN get_action_description(:send_message) called THEN mentions announcement for recursive broadcast - description = Schema.get_action_description(:send_message) - - assert is_binary(description) - assert description =~ "announcement" - # Should mention recursive or descendants - assert description =~ ~r/(recursive|descendant|subtree|all)/i - end - - # R3: Children vs Announcement Distinguished - test "send_message description distinguishes children from announcement" do - # [UNIT] - WHEN get_action_description(:send_message) called THEN clarifies children is direct only - description = Schema.get_action_description(:send_message) - - assert is_binary(description) - # Should clarify that children means direct children only (both concepts must be present) - assert description =~ "children" - assert description =~ "direct" - end - - # R4: One-Way Nature Documented - test "send_message description notes announcement is one-way" do - # [UNIT] - WHEN get_action_description(:send_message) called THEN mentions announcements are one-way - description = Schema.get_action_description(:send_message) - - assert is_binary(description) - # Should mention one-way nature or no reply expected - assert description =~ ~r/(one-way|broadcast|no reply)/i - end - - # R5: Prompt Integration - test "system prompt includes announcement target documentation" do - # [INTEGRATION] - WHEN build_system_prompt called THEN generated prompt includes announcement documentation - prompt = PromptBuilder.build_system_prompt(capability_groups: @all_capability_groups) - - # The system prompt should document the announcement target - assert prompt =~ "announcement" - - # Should explain its purpose (recursive broadcast to descendants) - assert prompt =~ ~r/(descendant|recursive|subtree)/i - - # Should distinguish from regular children target - assert prompt =~ ~r/children/i - end - end end diff --git a/test/quoracle/consensus/result_batch_sync_test.exs b/test/quoracle/consensus/result_batch_sync_test.exs index 454c7c1..060badb 100644 --- a/test/quoracle/consensus/result_batch_sync_test.exs +++ b/test/quoracle/consensus/result_batch_sync_test.exs @@ -5,7 +5,7 @@ defmodule Quoracle.Consensus.ResultBatchSyncTest do Packet: 2 (Consensus Logic) """ - use Quoracle.DataCase, async: true + use ExUnit.Case, async: true alias Quoracle.Consensus.Result # ============================================================================= diff --git a/test/quoracle/costs/aggregator_test.exs b/test/quoracle/costs/aggregator_test.exs index e25bf92..d0c2c68 100644 --- a/test/quoracle/costs/aggregator_test.exs +++ b/test/quoracle/costs/aggregator_test.exs @@ -36,6 +36,18 @@ defmodule Quoracle.Costs.AggregatorTest do - R26: Request Count Accurate [INTEGRATION] - R27: Nil Cost Records Handled [INTEGRATION] - R28: UUID Binary Conversion [UNIT] + + Requirements (v3.0 - fix-20260223-cost-display-budget-timeout): + - R29: Subtree Per-Model Query [INTEGRATION] + - R30: Multi-Agent Per-Model Query [INTEGRATION] + - R31: Includes NULL Model Spec Group [INTEGRATION] + - R32: Empty Agent List [UNIT] + - R33: Leaf Agent Matches Single Agent [INTEGRATION] + - R34: Token Counts Summed Across Subtree [INTEGRATION] + - R35: Property — Subtree Total Equals Agent + Children [UNIT] + + Type Contract Fix (fix-20260223-cost-display-budget-timeout): + - R36: model_cost_detailed type allows nil model_spec [UNIT] """ use Quoracle.DataCase, async: true @@ -1327,4 +1339,440 @@ defmodule Quoracle.Costs.AggregatorTest do assert length(result) == 1 end end + + # ============================================================ + # COST_Aggregator v3.0: R29 - Subtree Per-Model Query [INTEGRATION] + # ============================================================ + + describe "agent_tree_detailed - subtree" do + test "R29: returns per-model costs for entire subtree (self + descendants)" do + task = create_task() + tree = create_agent_tree(task) + + # Root uses model A + create_detailed_cost(task, tree.root.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.10"), + input_tokens: 500, + output_tokens: 200 + ) + + # Child1 uses model A and model B + create_detailed_cost(task, tree.child1.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.15"), + input_tokens: 800, + output_tokens: 300 + ) + + create_detailed_cost(task, tree.child1.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.08"), + input_tokens: 400, + output_tokens: 150 + ) + + # Grandchild uses model B + create_detailed_cost(task, tree.grandchild.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.12"), + input_tokens: 600, + output_tokens: 250 + ) + + result = Aggregator.by_agent_tree_and_model_detailed(tree.root.agent_id) + + # Should return 2 model groups: claude-sonnet-4 and gpt-4o + assert length(result) == 2 + + claude = Enum.find(result, &(&1.model_spec == "anthropic/claude-sonnet-4")) + gpt = Enum.find(result, &(&1.model_spec == "openai/gpt-4o")) + + # Claude: root (0.10) + child1 (0.15) = 0.25 + assert claude != nil + assert Decimal.equal?(claude.total_cost, Decimal.new("0.25")) + assert claude.request_count == 2 + + # GPT: child1 (0.08) + grandchild (0.12) = 0.20 + assert gpt != nil + assert Decimal.equal?(gpt.total_cost, Decimal.new("0.20")) + assert gpt.request_count == 2 + end + end + + # ============================================================ + # COST_Aggregator v3.0: R30 - Multi-Agent Per-Model Query [INTEGRATION] + # ============================================================ + + describe "agent_ids_detailed - multi-agent" do + test "R30: groups costs across multiple agents by model" do + task = create_task() + agent1 = create_agent(task) + agent2 = create_agent(task) + + # Both agents use the same model + create_detailed_cost(task, agent1.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.10"), + input_tokens: 500, + output_tokens: 200 + ) + + create_detailed_cost(task, agent2.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.20"), + input_tokens: 1000, + output_tokens: 400 + ) + + # Agent2 also uses a different model + create_detailed_cost(task, agent2.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.05"), + input_tokens: 300, + output_tokens: 100 + ) + + result = + Aggregator.by_agent_ids_and_model_detailed([ + agent1.agent_id, + agent2.agent_id + ]) + + assert length(result) == 2 + + claude = Enum.find(result, &(&1.model_spec == "anthropic/claude-sonnet-4")) + gpt = Enum.find(result, &(&1.model_spec == "openai/gpt-4o")) + + # Claude: agent1 (0.10) + agent2 (0.20) = 0.30, combined tokens + assert claude != nil + assert Decimal.equal?(claude.total_cost, Decimal.new("0.30")) + assert claude.request_count == 2 + assert claude.input_tokens == 1500 + assert claude.output_tokens == 600 + + # GPT: only agent2 + assert gpt != nil + assert Decimal.equal?(gpt.total_cost, Decimal.new("0.05")) + assert gpt.request_count == 1 + end + end + + # ============================================================ + # COST_Aggregator v3.0: R31 - Includes NULL Model Spec Group [INTEGRATION] + # ============================================================ + + describe "agent_ids_detailed - null model_spec" do + test "R31: includes costs without model_spec as nil group" do + task = create_task() + agent = create_agent(task) + + # Create cost WITH model_spec + create_detailed_cost(task, agent.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.10") + ) + + # Create cost WITHOUT model_spec (external cost) + {:ok, _cost} = + %AgentCost{} + |> AgentCost.changeset(%{ + agent_id: agent.agent_id, + task_id: task.id, + cost_type: "external", + cost_usd: Decimal.new("0.03"), + metadata: %{"source" => "web_fetch"} + }) + |> Repo.insert() + + result = Aggregator.by_agent_ids_and_model_detailed([agent.agent_id]) + + # Should return 2 groups: one with model_spec, one with nil + assert length(result) == 2 + + claude = Enum.find(result, &(&1.model_spec == "anthropic/claude-sonnet-4")) + nil_group = Enum.find(result, &is_nil(&1.model_spec)) + + assert claude != nil + assert Decimal.equal?(claude.total_cost, Decimal.new("0.10")) + + assert nil_group != nil + assert Decimal.equal?(nil_group.total_cost, Decimal.new("0.03")) + assert nil_group.request_count == 1 + end + end + + # ============================================================ + # COST_Aggregator v3.0: R32 - Empty Agent List [UNIT] + # ============================================================ + + describe "agent_ids_detailed - empty list" do + test "R32: empty agent list returns empty results" do + result = Aggregator.by_agent_ids_and_model_detailed([]) + + assert result == [] + end + end + + # ============================================================ + # COST_Aggregator v3.0: R33 - Single Agent Matches by_agent_and_model_detailed [INTEGRATION] + # ============================================================ + + describe "agent_tree_detailed - leaf agent" do + test "R33: leaf agent subtree matches single agent detailed query" do + task = create_task() + tree = create_agent_tree(task) + + # Only add costs to grandchild (a leaf) + create_detailed_cost(task, tree.grandchild.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.15"), + input_tokens: 700, + output_tokens: 300, + reasoning_tokens: 100, + cached_tokens: 50, + cache_creation_tokens: 25, + input_cost: "0.01", + output_cost: "0.02" + ) + + # Query via subtree (should just be the leaf's own costs) + subtree_result = Aggregator.by_agent_tree_and_model_detailed(tree.grandchild.agent_id) + + # Query via single agent detailed + single_result = Aggregator.by_agent_and_model_detailed(tree.grandchild.agent_id) + + # Both should return 1 model row + assert length(subtree_result) == 1 + assert length(single_result) == 1 + + subtree_model = hd(subtree_result) + single_model = hd(single_result) + + # Model-spec rows should match + assert subtree_model.model_spec == single_model.model_spec + assert Decimal.equal?(subtree_model.total_cost, single_model.total_cost) + assert subtree_model.request_count == single_model.request_count + assert subtree_model.input_tokens == single_model.input_tokens + assert subtree_model.output_tokens == single_model.output_tokens + assert subtree_model.reasoning_tokens == single_model.reasoning_tokens + assert subtree_model.cached_tokens == single_model.cached_tokens + assert subtree_model.cache_creation_tokens == single_model.cache_creation_tokens + end + end + + # ============================================================ + # COST_Aggregator v3.0: R34 - Token Counts Summed Across Subtree [INTEGRATION] + # ============================================================ + + describe "agent_tree_detailed - tokens" do + test "R34: token counts summed across subtree for same model" do + task = create_task() + tree = create_agent_tree(task) + + # Root: 500 input, 200 output, 100 reasoning + create_detailed_cost(task, tree.root.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.10"), + input_tokens: 500, + output_tokens: 200, + reasoning_tokens: 100, + cached_tokens: 50, + cache_creation_tokens: 25 + ) + + # Child1: 800 input, 300 output, 150 reasoning + create_detailed_cost(task, tree.child1.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.15"), + input_tokens: 800, + output_tokens: 300, + reasoning_tokens: 150, + cached_tokens: 60, + cache_creation_tokens: 30 + ) + + # Grandchild: 600 input, 250 output, 80 reasoning + create_detailed_cost(task, tree.grandchild.agent_id, + model_spec: "anthropic/claude-sonnet-4", + cost_usd: Decimal.new("0.12"), + input_tokens: 600, + output_tokens: 250, + reasoning_tokens: 80, + cached_tokens: 40, + cache_creation_tokens: 20 + ) + + result = Aggregator.by_agent_tree_and_model_detailed(tree.root.agent_id) + + assert length(result) == 1 + model = hd(result) + + # Token counts should be summed: root + child1 + grandchild + assert model.input_tokens == 500 + 800 + 600 + assert model.output_tokens == 200 + 300 + 250 + assert model.reasoning_tokens == 100 + 150 + 80 + assert model.cached_tokens == 50 + 60 + 40 + assert model.cache_creation_tokens == 25 + 30 + 20 + assert model.request_count == 3 + end + end + + # ============================================================ + # COST_Aggregator v3.0: R35 - Property: Subtree Total Equals Agent + Children [UNIT] + # ============================================================ + + describe "subtree total consistency" do + property "R35: subtree model total equals agent + children total" do + check all( + root_cost_cents <- integer(1..1000), + child_cost_cents <- integer(1..1000), + grandchild_cost_cents <- integer(1..1000) + ) do + task = create_task() + tree = create_agent_tree(task) + + root_cost = Decimal.div(Decimal.new(root_cost_cents), 100) + child_cost = Decimal.div(Decimal.new(child_cost_cents), 100) + grandchild_cost = Decimal.div(Decimal.new(grandchild_cost_cents), 100) + + create_detailed_cost(task, tree.root.agent_id, + model_spec: "test/model", + cost_usd: root_cost + ) + + create_detailed_cost(task, tree.child1.agent_id, + model_spec: "test/model", + cost_usd: child_cost + ) + + create_detailed_cost(task, tree.grandchild.agent_id, + model_spec: "test/model", + cost_usd: grandchild_cost + ) + + # Get subtree per-model total + subtree_result = Aggregator.by_agent_tree_and_model_detailed(tree.root.agent_id) + + subtree_total = + Enum.reduce(subtree_result, Decimal.new("0"), fn row, acc -> + if row.total_cost, do: Decimal.add(acc, row.total_cost), else: acc + end) + + # Get scalar totals via existing functions + own_result = Aggregator.by_agent(tree.root.agent_id) + children_result = Aggregator.by_agent_children(tree.root.agent_id) + + own_total = own_result.total_cost || Decimal.new("0") + children_total = children_result.total_cost || Decimal.new("0") + expected_total = Decimal.add(own_total, children_total) + + assert Decimal.equal?(subtree_total, expected_total), + "Subtree per-model sum (#{subtree_total}) should equal " <> + "by_agent (#{own_total}) + by_agent_children (#{children_total}) = #{expected_total}" + end + end + end + + # ============================================================ + # Type Contract Fix: model_cost_detailed allows nil model_spec + # WorkGroupID: fix-20260223-cost-display-budget-timeout + # Spec: COST_Aggregator v3.0 type definition (line 511: model_spec: String.t() | nil) + # ============================================================ + + describe "v3.0: model_cost_detailed type contract" do + test "model_cost_detailed @type includes nil for model_spec" do + # [UNIT] - The model_cost_detailed @type must allow model_spec: String.t() | nil + # because by_agent_ids_and_model_detailed (v3.0) includes costs with NULL model_spec + # (e.g., external costs) as a nil group. + # + # This test introspects the compiled type to verify the annotation is correct. + # Currently the type says model_spec: String.t() — this FAILS because nil + # is not in the union. + + {:ok, types} = Code.Typespec.fetch_types(Quoracle.Costs.Aggregator) + + # Find the model_cost_detailed type definition + detailed_type = + Enum.find(types, fn + {:type, {:model_cost_detailed, _def, _args}} -> true + _ -> false + end) + + assert detailed_type != nil, "model_cost_detailed type not found in module" + + {:type, {:model_cost_detailed, type_def, _args}} = detailed_type + + # Extract the model_spec field from the map type + {:type, _line, :map, fields} = type_def + + model_spec_field = + Enum.find(fields, fn + {:type, _, :map_field_exact, [{:atom, _, :model_spec} | _]} -> true + _ -> false + end) + + assert model_spec_field != nil, "model_spec field not found in type" + + {:type, _, :map_field_exact, [_key, value_type]} = model_spec_field + + # The value type must be a union that includes nil + # Currently it's {:remote_type, _, [{:atom, 0, String}, {:atom, 0, :t}, []]} + # After fix it should be {:type, _, :union, [String.t(), nil]} + assert match?({:type, _, :union, _}, value_type), + "model_spec type must be a union (String.t() | nil), " <> + "got: #{inspect(value_type)}" + + {:type, _, :union, union_members} = value_type + + has_nil = + Enum.any?(union_members, fn + {:atom, _, nil} -> true + _ -> false + end) + + assert has_nil, + "model_spec union type must include nil, " <> + "got members: #{inspect(union_members)}" + end + + # TEST-FIX: Runtime regression guard — deferred from TEST phase because it passes + # against current code (the function already works, only the @type was wrong). + test "by_agent_ids_and_model_detailed returns nil model_spec for null group" do + # [INTEGRATION] - Costs without model_spec in metadata appear as nil group + task = create_task() + agent = create_agent(task) + + # Cost WITH model_spec + create_cost(task, agent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.10") + ) + + # Cost WITHOUT model_spec (e.g., external cost) + {:ok, _} = + %AgentCost{} + |> AgentCost.changeset(%{ + agent_id: agent.agent_id, + task_id: task.id, + cost_type: "external", + cost_usd: Decimal.new("0.03"), + metadata: %{"description" => "no model_spec here"} + }) + |> Repo.insert() + + results = Aggregator.by_agent_ids_and_model_detailed([agent.agent_id]) + + assert length(results) == 2 + + nil_group = Enum.find(results, &is_nil(&1.model_spec)) + assert nil_group != nil, "Expected a nil model_spec group for costs without model_spec" + assert nil_group.request_count == 1 + assert Decimal.equal?(nil_group.total_cost, Decimal.new("0.03")) + + named_group = Enum.find(results, &(&1.model_spec == "anthropic/claude-sonnet-4-20250514")) + assert named_group != nil + assert named_group.request_count == 1 + end + end end diff --git a/test/quoracle/persistence_resilience_test.exs b/test/quoracle/persistence_resilience_test.exs index 949a6c8..2568446 100644 --- a/test/quoracle/persistence_resilience_test.exs +++ b/test/quoracle/persistence_resilience_test.exs @@ -167,6 +167,7 @@ defmodule Quoracle.PersistenceResilienceTest do agent_id: agent_id, task_id: task.id, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, @@ -224,6 +225,7 @@ defmodule Quoracle.PersistenceResilienceTest do agent_id: agent_id, task_id: task.id, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, @@ -345,6 +347,7 @@ defmodule Quoracle.PersistenceResilienceTest do agent_id: agent_id, task_id: task.id, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, @@ -692,6 +695,7 @@ defmodule Quoracle.PersistenceResilienceTest do agent_id: agent_id, task_id: task.id, test_mode: true, + force_persist: true, sandbox_owner: sandbox_owner, pubsub: deps.pubsub, registry: deps.registry, diff --git a/test/quoracle/providers/retry_helper_test.exs b/test/quoracle/providers/retry_helper_test.exs index 09c0100..a4cc121 100644 --- a/test/quoracle/providers/retry_helper_test.exs +++ b/test/quoracle/providers/retry_helper_test.exs @@ -15,6 +15,10 @@ defmodule Quoracle.Providers.RetryHelperTest do - R8: Retry FunctionClauseError with 429 error in stacktrace args (v3.2) - R9: Retry FunctionClauseError with 5xx error in stacktrace args (v3.2) - R10: FunctionClauseError without HTTP error still returns :malformed_response (v3.2) + - R11: Retry FunctionClauseError with string throttling message in error map (v3.3) + - R12: Retry FunctionClauseError with string error body containing throttling keywords (v3.3) + - R13: String error body with unavailable/overloaded keywords maps to 503 (v3.3) + - R14: String error body without recognizable keywords still returns :malformed_response (v3.3) """ use ExUnit.Case, async: true @@ -651,6 +655,306 @@ defmodule Quoracle.Providers.RetryHelperTest do end end + describe "R11: Retry string throttle in error map" do + test "retries when provider crashes with string error containing 'throttled'" do + attempts = :counters.new(1, [:atomics]) + delay_calls = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + # Simulate Google Vertex throttling: {"error": "The request is throttled..."} + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "The request is throttled due to too many concurrent requests."}, + :model, + :context + ] + ) + else + {:ok, "recovered from throttle"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :counters.add(delay_calls, 1, 1) end + )} + ) + end) + + assert_received {:result, {:ok, "recovered from throttle"}} + assert :counters.get(delay_calls, 1) >= 1 + end + + test "retries when error message contains 'rate limit'" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "Rate limit exceeded for this model"}, + :model, + :context + ] + ) + else + {:ok, "recovered"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:ok, "recovered"}} + end + + test "retries when error message contains 'too many'" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "Too many requests, please slow down"}, + :model, + :context + ] + ) + else + {:ok, "recovered"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:ok, "recovered"}} + end + end + + describe "R12: Retry raw string throttle body" do + test "retries when raw string body contains throttling keyword" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + # Simulate undecoded string body passed to parse_response + apply( + __MODULE__, + :simulate_parse_response, + [ + ~s({"error":"The request is throttled due to too many concurrent requests."}), + :model, + :context + ] + ) + else + {:ok, "recovered from raw string throttle"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:ok, "recovered from raw string throttle"}} + end + end + + describe "R13: String error maps to 503" do + test "retries when error message contains 'overloaded'" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "The model is currently overloaded"}, + :model, + :context + ] + ) + else + {:ok, "recovered from overload"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:ok, "recovered from overload"}} + end + + test "retries when error message contains 'unavailable'" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + attempt = :counters.get(attempts, 1) + :counters.add(attempts, 1, 1) + + if attempt < 2 do + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "Service temporarily unavailable"}, + :model, + :context + ] + ) + else + {:ok, "recovered from unavailable"} + end + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:ok, "recovered from unavailable"}} + end + end + + describe "R14: Unrecognized string not retried" do + test "string error without recognizable keywords is not retried" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + :counters.add(attempts, 1, 1) + + apply( + __MODULE__, + :simulate_parse_response, + [ + %{"error" => "Invalid request format"}, + :model, + :context + ] + ) + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:error, :malformed_response}} + assert :counters.get(attempts, 1) == 1 + end + + test "raw string body without recognizable keywords is not retried" do + attempts = :counters.new(1, [:atomics]) + + func = fn -> + :counters.add(attempts, 1, 1) + + apply( + __MODULE__, + :simulate_parse_response, + [ + ~s({"error":"Something completely unexpected"}), + :model, + :context + ] + ) + end + + capture_log(fn -> + send( + self(), + {:result, + RetryHelper.with_retry(func, + initial_delay: 10, + error_module: @error_module, + delay_fn: fn _ms -> :ok end + )} + ) + end) + + assert_received {:result, {:error, :malformed_response}} + assert :counters.get(attempts, 1) == 1 + end + end + # Helper function that always raises FunctionClauseError. # When called via apply/4, the args appear in __STACKTRACE__ for extraction. # Uses pattern matching that will never match, forcing the FunctionClauseError. diff --git a/test/quoracle/tasks/task_manager_budget_test.exs b/test/quoracle/tasks/task_manager_budget_test.exs index 3c02956..cd0daa5 100644 --- a/test/quoracle/tasks/task_manager_budget_test.exs +++ b/test/quoracle/tasks/task_manager_budget_test.exs @@ -98,7 +98,7 @@ defmodule Quoracle.Tasks.TaskManagerBudgetTest do {:ok, state} = Core.get_state(agent_pid) assert state.budget_data.mode == :na assert state.budget_data.allocated == nil - assert state.budget_data.committed == nil + assert Decimal.equal?(state.budget_data.committed, Decimal.new(0)) register_agent_cleanup(agent_pid, cleanup_tree: true, registry: deps.registry) end diff --git a/test/quoracle/tasks/task_restorer_test.exs b/test/quoracle/tasks/task_restorer_test.exs index 313bea2..0c39b90 100644 --- a/test/quoracle/tasks/task_restorer_test.exs +++ b/test/quoracle/tasks/task_restorer_test.exs @@ -35,7 +35,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -89,7 +90,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -143,7 +145,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Monitor agent BEFORE pause (Task.start fire-and-forget requires waiting) @@ -181,7 +184,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Get agent_id for registry lookup @@ -214,7 +218,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Spawn agent tree: root -> child1 -> child2 (with automatic cleanup) @@ -313,7 +318,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Spawn and terminate agent first (with automatic cleanup) @@ -327,7 +333,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do status: "running", task: "Temp", test_mode: true, - test_opts: [skip_initial_consultation: true] + test_opts: [skip_initial_consultation: true], + force_persist: true }, registry: registry, pubsub: pubsub, @@ -359,7 +366,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Spawn two agents (with automatic cleanup) @@ -412,7 +420,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, pid} = @@ -467,7 +476,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # TODO: Need DynSup.get_dynsup_pid/0 to return nil for testing @@ -488,7 +498,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Create agent records in database (simulating paused state) @@ -569,7 +580,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Create parent and child in database @@ -642,7 +654,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _root_db} = @@ -696,7 +709,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Terminate the root agent so we can test restoration @@ -728,7 +742,7 @@ defmodule Quoracle.Tasks.TaskRestorerTest do task_id: task.id, status: "running", parent_id: nil, - config: %{test_mode: true}, + config: %{test_mode: true, force_persist: true}, inserted_at: ~N[2025-01-01 10:00:00] }) @@ -787,7 +801,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _root_db} = @@ -845,7 +860,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Pause to remove agents @@ -870,7 +886,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _root_db} = @@ -921,7 +938,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _agent} = @@ -953,7 +971,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, root_pid} = @@ -1068,7 +1087,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Monitor BEFORE pause (to catch fast terminations) @@ -1107,7 +1127,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Inject model_histories via :sys.replace_state (simulates conversation) @@ -1196,7 +1217,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, pid} = @@ -1233,7 +1255,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Create child without parent in DB (orphan) @@ -1286,7 +1309,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Both agents will succeed (empty config is valid) @@ -1296,7 +1320,7 @@ defmodule Quoracle.Tasks.TaskRestorerTest do task_id: task.id, status: "running", parent_id: nil, - config: %{test_mode: true}, + config: %{test_mode: true, force_persist: true}, inserted_at: ~N[2025-01-01 10:00:00] }) @@ -1366,7 +1390,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -1426,7 +1451,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -1482,7 +1508,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -1583,7 +1610,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent before restore @@ -1694,7 +1722,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent before restore @@ -1783,7 +1812,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent before restore @@ -1870,7 +1900,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent and mark its DB record as stopped so it's excluded @@ -1929,7 +1960,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent @@ -2030,7 +2062,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -2095,7 +2128,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = @@ -2147,7 +2181,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent @@ -2236,7 +2271,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent @@ -2349,7 +2385,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Stop task agent @@ -2427,7 +2464,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, root_pid} = @@ -2439,6 +2477,7 @@ defmodule Quoracle.Tasks.TaskRestorerTest do parent_pid: nil, status: "running", task: "Root agent", + force_persist: true, sandbox_owner: sandbox_owner }, registry: registry, @@ -2455,6 +2494,7 @@ defmodule Quoracle.Tasks.TaskRestorerTest do parent_pid: root_pid, status: "running", task: "First child", + force_persist: true, sandbox_owner: sandbox_owner }, registry: registry, @@ -2471,6 +2511,7 @@ defmodule Quoracle.Tasks.TaskRestorerTest do parent_pid: root_pid, status: "running", task: "Second child", + force_persist: true, sandbox_owner: sandbox_owner }, registry: registry, @@ -2613,7 +2654,8 @@ defmodule Quoracle.Tasks.TaskRestorerTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_pid} = diff --git a/test/quoracle_web/live/dashboard_3panel_integration_test.exs b/test/quoracle_web/live/dashboard_3panel_integration_test.exs index f0f85e7..f2f66df 100644 --- a/test/quoracle_web/live/dashboard_3panel_integration_test.exs +++ b/test/quoracle_web/live/dashboard_3panel_integration_test.exs @@ -37,6 +37,7 @@ defmodule QuoracleWeb.Dashboard3PanelIntegrationTest do registry: registry_name, dynsup: dynsup_name, sandbox_owner: sandbox_owner, + force_persist: true, profile: profile } end diff --git a/test/quoracle_web/live/dashboard_live_test.exs b/test/quoracle_web/live/dashboard_live_test.exs index ac8d3c4..b6ef58b 100644 --- a/test/quoracle_web/live/dashboard_live_test.exs +++ b/test/quoracle_web/live/dashboard_live_test.exs @@ -250,7 +250,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -296,7 +297,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -561,7 +563,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -626,7 +629,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -665,7 +669,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -805,7 +810,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -850,7 +856,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -933,7 +940,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, {task2, _pid2}} = @@ -941,7 +949,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Mount Dashboard @@ -1019,7 +1028,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(task.id, "paused") @@ -1068,7 +1078,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(task.id, "paused") @@ -1234,7 +1245,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -1306,7 +1318,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(task.id, "paused") @@ -1375,7 +1388,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(task.id, "paused") @@ -1434,7 +1448,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Use same pattern as working tests - capture_log + send/receive @@ -1496,7 +1511,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(task.id, "paused") @@ -1579,7 +1595,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _view, html} = @@ -1615,7 +1632,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, {paused_task, _pid2}} = @@ -1623,7 +1641,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(paused_task.id, "paused") @@ -1633,7 +1652,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(completed_task.id, "completed") @@ -1643,7 +1663,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) Quoracle.Tasks.TaskManager.update_task_status(failed_task.id, "failed") @@ -1679,7 +1700,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, _view, html} = @@ -1724,7 +1746,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Get agent_id from the spawned agent @@ -1769,7 +1792,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_state} = GenServer.call(agent_pid, :get_state) @@ -1969,7 +1993,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -2021,7 +2046,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, agent_state} = GenServer.call(agent_pid, :get_state) @@ -2107,7 +2133,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -2191,7 +2218,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -2283,7 +2311,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, view, _html} = @@ -2386,7 +2415,8 @@ defmodule QuoracleWeb.DashboardLiveTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) {:ok, root_state} = GenServer.call(root_pid, :get_state) @@ -2405,6 +2435,7 @@ defmodule QuoracleWeb.DashboardLiveTest do # parent_pid needed for persistence to look up parent agent_id via Registry parent_pid: root_pid, test_mode: true, + force_persist: true, # sandbox_owner must be in config map (not opts) for DB persistence sandbox_owner: sandbox_owner }, @@ -2972,7 +3003,8 @@ defmodule QuoracleWeb.DashboardLiveTest do agent_id: agent_id, task_id: task.id, task_description: "Test", - test_mode: true + test_mode: true, + force_persist: true }, registry: registry, pubsub: pubsub, diff --git a/test/quoracle_web/live/dashboard_pause_resume_integration_test.exs b/test/quoracle_web/live/dashboard_pause_resume_integration_test.exs index e7b2217..a4a4ae5 100644 --- a/test/quoracle_web/live/dashboard_pause_resume_integration_test.exs +++ b/test/quoracle_web/live/dashboard_pause_resume_integration_test.exs @@ -62,7 +62,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) # Wait for initialization @@ -129,7 +130,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -208,7 +210,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -366,7 +369,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -446,7 +450,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -522,7 +527,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -571,7 +577,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -680,7 +687,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -796,7 +804,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) @@ -937,7 +946,8 @@ defmodule QuoracleWeb.DashboardPauseResumeIntegrationTest do sandbox_owner: sandbox_owner, dynsup: dynsup, registry: registry, - pubsub: pubsub + pubsub: pubsub, + force_persist: true ) assert {:ok, _state} = Quoracle.Agent.Core.get_state(task_agent_pid) diff --git a/test/quoracle_web/live/ui/cost_display_test.exs b/test/quoracle_web/live/ui/cost_display_test.exs index e5bd73b..36d9931 100644 --- a/test/quoracle_web/live/ui/cost_display_test.exs +++ b/test/quoracle_web/live/ui/cost_display_test.exs @@ -21,6 +21,11 @@ defmodule QuoracleWeb.UI.CostDisplayTest do - R13: Cost Update Handling [INTEGRATION] - R14: Agent Cost Display Flow [SYSTEM] - R15: Task Total Display Flow [SYSTEM] + - R16-R19: ID Attribute & costs_updated_at Trigger [UNIT/INTEGRATION] + - R20-R39: v2.0 Token Breakdown Table [UNIT/INTEGRATION] + - R40: Absorption Records in Model Breakdown Table [ACCEPTANCE] + - R41: Header Total Stable After Child Dismissal [ACCEPTANCE] + - R42: Model Table Row Sum Equals Header Total [ACCEPTANCE] """ use QuoracleWeb.ConnCase, async: true @@ -2154,4 +2159,233 @@ defmodule QuoracleWeb.UI.CostDisplayTest do assert html =~ "$0.05" end end + + # ============================================================ + # WorkGroupID: fix-20260223-cost-display-budget-timeout + # v3.0: Absorption Records from DismissChild (R40-R42) + # ============================================================ + + # Helper to create an absorption cost record (simulates DismissChild v5.0 per-model absorption) + defp create_absorption_record(task, parent_agent_id, opts) do + child_id = Keyword.get(opts, :child_id, "child_#{System.unique_integer([:positive])}") + cost_usd = Keyword.get(opts, :cost_usd, Decimal.new("0.05")) + model_spec = Keyword.get(opts, :model_spec, "anthropic/claude-sonnet-4-20250514") + input_tokens = Keyword.get(opts, :input_tokens, 1000) + output_tokens = Keyword.get(opts, :output_tokens, 500) + input_cost = Keyword.get(opts, :input_cost, "0.02") + output_cost = Keyword.get(opts, :output_cost, "0.03") + + metadata = + %{ + "child_agent_id" => child_id, + "child_allocated" => "50", + "child_tree_spent" => Decimal.to_string(cost_usd), + "unspent_returned" => "0", + "dismissed_at" => DateTime.to_iso8601(DateTime.utc_now()), + "model_spec" => model_spec, + "input_tokens" => to_string(input_tokens), + "output_tokens" => to_string(output_tokens), + "reasoning_tokens" => "0", + "cached_tokens" => "0", + "cache_creation_tokens" => "0", + "input_cost" => input_cost, + "output_cost" => output_cost + } + + {:ok, cost} = + %AgentCost{} + |> AgentCost.changeset(%{ + agent_id: parent_agent_id, + task_id: task.id, + cost_type: "child_budget_absorbed", + cost_usd: cost_usd, + metadata: metadata + }) + |> Repo.insert() + + cost + end + + describe "R40: absorption records appear in model breakdown table" do + test "dismissed child costs appear in model breakdown table", %{ + conn: conn, + sandbox_owner: sandbox_owner + } do + task = create_task() + parent = create_agent(task) + + # Parent's own cost (direct LLM usage) + create_detailed_cost(task, parent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.10"), + input_tokens: 2000, + output_tokens: 500 + ) + + # Absorption record from dismissed child (simulates DismissChild v5.0) + # This record is attributed to the parent but represents absorbed child costs + create_absorption_record(task, parent.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.20"), + input_tokens: 3000, + output_tokens: 1000, + input_cost: "0.08", + output_cost: "0.12" + ) + + {:ok, view, _html} = + render_isolated( + conn, + %{ + id: "cost-absorption-r40-#{task.id}", + mode: :detail, + task_id: task.id, + expanded: true + }, + sandbox_owner + ) + + html = render(view) + + # The model breakdown table should show BOTH models: + # - claude-sonnet from parent's own LLM usage + # - gpt-4o from the absorbed child costs + assert html =~ "claude-sonnet" + assert html =~ "gpt-4o" + + # Absorbed child cost should appear with its cost value + assert html =~ "$0.20" + end + end + + describe "R41: header total stable after dismissal" do + test "Cost Details header total unchanged after child dismissal", %{ + conn: conn, + sandbox_owner: sandbox_owner + } do + task = create_task() + parent = create_agent(task) + + # Parent's own cost + create_detailed_cost(task, parent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.15"), + input_tokens: 2000, + output_tokens: 500 + ) + + # Absorption records simulate the scenario AFTER child dismissal: + # - Child's original cost records have been deleted by TreeTerminator + # - These absorption records under the parent preserve the costs + create_absorption_record(task, parent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.25"), + input_tokens: 5000, + output_tokens: 1500, + input_cost: "0.10", + output_cost: "0.15" + ) + + # Expected total: parent own ($0.15) + absorbed child ($0.25) = $0.40 + {:ok, view, _html} = + render_isolated( + conn, + %{ + id: "cost-absorption-r41-#{task.id}", + mode: :detail, + task_id: task.id, + expanded: true + }, + sandbox_owner + ) + + html = render(view) + + # The header total should include both own and absorbed costs + # by_task total = $0.15 + $0.25 = $0.40 + assert html =~ "$0.40" + end + end + + describe "R42: model table row sum equals header total" do + test "model table row sum equals header total", %{ + conn: conn, + sandbox_owner: sandbox_owner + } do + task = create_task() + parent = create_agent(task) + + # Parent's own costs across 2 models + create_detailed_cost(task, parent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.10"), + input_tokens: 2000, + output_tokens: 500, + input_cost: "0.04", + output_cost: "0.06" + ) + + create_detailed_cost(task, parent.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.05"), + input_tokens: 1000, + output_tokens: 300, + input_cost: "0.02", + output_cost: "0.03" + ) + + # Absorption records from dismissed child (2 different models) + create_absorption_record(task, parent.agent_id, + model_spec: "anthropic/claude-sonnet-4-20250514", + cost_usd: Decimal.new("0.15"), + input_tokens: 3000, + output_tokens: 800, + input_cost: "0.06", + output_cost: "0.09" + ) + + create_absorption_record(task, parent.agent_id, + model_spec: "openai/gpt-4o", + cost_usd: Decimal.new("0.20"), + input_tokens: 4000, + output_tokens: 1200, + input_cost: "0.08", + output_cost: "0.12" + ) + + {:ok, view, _html} = + render_isolated( + conn, + %{ + id: "cost-absorption-r42-#{task.id}", + mode: :detail, + task_id: task.id, + expanded: true + }, + sandbox_owner + ) + + html = render(view) + + # Header total comes from by_task which sums ALL cost records: + # $0.10 + $0.05 + $0.15 + $0.20 = $0.50 + assert html =~ "$0.50" + + # Model table rows (aggregated per model_spec from by_task_and_model_detailed): + # claude-sonnet: $0.10 + $0.15 = $0.25 + assert html =~ "$0.25" + # gpt-4o: $0.05 + $0.20 = $0.25 + # (both models sum to $0.25, which equals $0.50 total) + + # Both models should be visible in the table + assert html =~ "claude-sonnet" + assert html =~ "gpt-4o" + + # Verify that both models appear and the header total ($0.50) equals + # the sum of model rows ($0.25 + $0.25 = $0.50). + # The header uses by_task (unfiltered) and model rows use + # by_task_and_model_detailed (filtered on model_spec IS NOT NULL). + # With absorption records preserving model_spec, these should match. + end + end end diff --git a/test/support/pubsub_isolation_test.exs b/test/support/pubsub_isolation_test.exs index bd17299..81391ba 100644 --- a/test/support/pubsub_isolation_test.exs +++ b/test/support/pubsub_isolation_test.exs @@ -79,32 +79,6 @@ defmodule Test.PubSubIsolationTest do # Should NOT receive duplicate refute_receive {:from_pubsub2, _}, 100 end - - test "messages don't leak to global PubSub" do - {:ok, isolated} = PubSubIsolation.setup_isolated_pubsub() - - # Subscribe to global PubSub - Phoenix.PubSub.subscribe(Quoracle.PubSub, "test_topic") - - # Broadcast to isolated instance - Phoenix.PubSub.broadcast(isolated, "test_topic", {:isolated_msg, "data"}) - - # Should NOT receive on global - refute_receive {:isolated_msg, _}, 100 - end - - test "global broadcasts don't reach isolated instances" do - {:ok, isolated} = PubSubIsolation.setup_isolated_pubsub() - - # Subscribe to isolated instance - Phoenix.PubSub.subscribe(isolated, "test_topic") - - # Broadcast to global PubSub - Phoenix.PubSub.broadcast(Quoracle.PubSub, "test_topic", {:global_msg, "data"}) - - # Should NOT receive on isolated - refute_receive {:global_msg, _}, 100 - end end describe "concurrent test isolation" do @@ -183,36 +157,7 @@ defmodule Test.PubSubIsolationTest do end end - describe "cleanup and resource management" do - test "can create many instances without resource issues" do - # Create 100 isolated instances - instances = - for _ <- 1..100 do - {:ok, pubsub} = PubSubIsolation.setup_isolated_pubsub() - pubsub - end - - # All should be unique - assert length(instances) == length(Enum.uniq(instances)) - - # All should be functional - Enum.each(instances, fn pubsub -> - assert :ok = Phoenix.PubSub.subscribe(pubsub, "mass_test") - end) - end - end - describe "backward compatibility" do - test "old tests using global PubSub still work" do - # Old tests don't call setup_isolated_pubsub - # They use the global Quoracle.PubSub directly - - Phoenix.PubSub.subscribe(Quoracle.PubSub, "global_topic") - Phoenix.PubSub.broadcast(Quoracle.PubSub, "global_topic", {:global, "test"}) - - assert_receive {:global, "test"} - end - test "explicit passing works alongside Process dictionary" do {:ok, pubsub} = PubSubIsolation.setup_isolated_pubsub()