From 4d84e570b71abdde7c99ad6d7b55885ae2df6f4f Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Sun, 17 May 2026 19:43:49 -0300 Subject: [PATCH] Evals (Layer 1): payload snapshot tests for prompt regressions --- .github/workflows/tests.yml | 8 + composer.json | 1 + docs/dev/evals.md | 92 +++++++ phpunit.xml.dist | 9 + .../PromptAssemblySnapshotTest.php | 233 ++++++++++++++++++ .../PromptPayloadAssembler.php | 177 +++++++++++++ .../__snapshots__/coordinator--chat.json | 62 +++++ .../__snapshots__/example--whatsapp.json | 29 +++ .../__snapshots__/loop-demo--chat.json | 33 +++ .../site-introspection--whatsapp.json | 69 ++++++ .../__snapshots__/workflow-drafter--chat.json | 24 ++ .../integration/prompt-assembly/bootstrap.php | 150 +++++++++++ .../stubs/function-declaration-stub.php | 23 ++ 13 files changed, 910 insertions(+) create mode 100644 docs/dev/evals.md create mode 100644 tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php create mode 100644 tests/integration/prompt-assembly/PromptPayloadAssembler.php create mode 100644 tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json create mode 100644 tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json create mode 100644 tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json create mode 100644 tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json create mode 100644 tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json create mode 100644 tests/integration/prompt-assembly/bootstrap.php create mode 100644 tests/integration/prompt-assembly/stubs/function-declaration-stub.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e56cf98..7e7bb2b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,6 +82,14 @@ jobs: - name: Run PHPUnit (unit suite) run: vendor/bin/phpunit --testsuite unit + - name: Run prompt-assembly snapshot suite + # Fails the build if the assembled provider payload (system prompt, + # tool catalog, model preference, runtime context) drifts from the + # committed snapshot for any canonical (agent, channel) pair. + # Update flow: `UPDATE_SNAPSHOTS=1 composer test:assembly` — the + # regenerated snapshot then goes through PR review. + run: composer test:assembly + - name: Run PHPUnit (integration suite) if: hashFiles('tests/integration/**') != '' run: vendor/bin/phpunit --testsuite integration diff --git a/composer.json b/composer.json index 274f73e..85d2640 100644 --- a/composer.json +++ b/composer.json @@ -39,6 +39,7 @@ "scripts": { "test": "phpunit", "test:smoke": "php tests/smoke.php", + "test:assembly": "phpunit --testsuite assembly", "lint:php": "phpcs", "lint:php:fix": "phpcbf", "analyse": "phpstan analyse" diff --git a/docs/dev/evals.md b/docs/dev/evals.md new file mode 100644 index 0000000..54a3926 --- /dev/null +++ b/docs/dev/evals.md @@ -0,0 +1,92 @@ +# Prompt evals + +openclaWP guards against prompt regressions in two complementary layers. Layer 1 +is shipped today; Layer 2 is queued. Tracking issue: +[#46](https://github.com/lezama/openclawp/issues/46). + +## Layer 1 — payload snapshot tests (shipping) + +Goal: catch unintentional drift in the provider payload (system prompt, message +list, tool catalog, model preference) at zero token cost. + +For each canonical `(agent, channel)` pair the suite freezes a user message, +asks the production code to build the request payload openclaWP would send to +the provider, and diffs that payload against a committed JSON snapshot under +`tests/integration/prompt-assembly/__snapshots__/`. + +Pairs currently covered: + +| Snapshot | Agent | Channel | +|-------------------------------------|----------------------------------|------------| +| `loop-demo--chat.json` | `openclawp-loop-demo` | `chat` | +| `site-introspection--whatsapp.json` | `openclawp-site-introspection` | `whatsapp` | +| `coordinator--chat.json` | `openclawp-coordinator` | `chat` | +| `workflow-drafter--chat.json` | `openclawp-workflow-drafter` | `chat` | +| `example--whatsapp.json` | `openclawp-example` | `whatsapp` | + +### Run + +```bash +composer test:assembly +``` + +Runs on every CI build inside the existing `phpunit` job (`.github/workflows/tests.yml`). +Adds <1s to the test cycle — no model calls, no network. + +### Failure mode + +A failing snapshot prints a unified diff of expected vs actual JSON and points +at the regenerate command. Example: + +``` +Failed asserting that two strings are identical. +--- Expected ++++ Actual +@@ @@ +- "system_instruction": "You are a precise assistant. …", ++ "system_instruction": "You are a regressed assistant. …", +``` + +### Updating snapshots (intentional changes) + +When you change a system prompt, tool description, or default config on +purpose, regenerate the affected snapshots and commit the diff: + +```bash +UPDATE_SNAPSHOTS=1 composer test:assembly +git add tests/integration/prompt-assembly/__snapshots__ +``` + +The PR reviewer reads the diff to confirm the change is intentional. Treat +every snapshot churn the same way you'd treat a copy edit on a customer-facing +string. + +### Adding a new pair + +Add a row to `canonical_pairs()` in +`tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php`, then run: + +```bash +UPDATE_SNAPSHOTS=1 composer test:assembly +``` + +The first run creates the snapshot. Subsequent runs diff against it. + +### Implementation notes + +- The assembler lives in + `tests/integration/prompt-assembly/PromptPayloadAssembler.php`. It mirrors + `OpenclaWP_Runner::build_turn_runner()` minus the provider call — same + agent description, same transcript, same `OpenclaWP_Tools_Resolver` output, + same model-preference resolver. +- JSON is pretty-printed with sorted keys (recursive on assoc arrays, preserving + list order). Trailing newline. No machine-specific paths. Tabs for indent so + the files match the repo's PHP/JS style. +- Zero new composer deps. Plain PHPUnit + `file_put_contents()` for updates. + +## Layer 2 — black-box conversation evals (deferred) + +YAML-defined eval suite covering 10 seed conversations across chat block, +WhatsApp, Telegram, and MCP, executed by promptfoo's HTTP provider against a +booted wp-env. Tracked in +[#49](https://github.com/lezama/openclawp/issues/49). diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 1ec5ae0..cf40239 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -10,5 +10,14 @@ tests/unit + + tests/integration + + tests/integration/prompt-assembly + + + tests/integration/prompt-assembly + diff --git a/tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php b/tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php new file mode 100644 index 0000000..06bbc0a --- /dev/null +++ b/tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php @@ -0,0 +1,233 @@ + true; + $GLOBALS['openclawp_test_filters'] = array( + 'openclawp_register_loop_demo' => $true_filter, + 'openclawp_register_site_introspection' => $true_filter, + 'openclawp_register_workflow_drafter' => $true_filter, + 'openclawp_register_coordinator_demo' => $true_filter, + 'openclawp_register_example_agent' => $true_filter, + ); + + $this->register_canonical_abilities(); + + // Run the production registrar so the snapshot reflects the *real* + // agent args. Subagent-coordinator registers last; the registrar's + // own ordering (priority 20) is preserved here by call order. + OpenclaWP_Agent_Registrar::maybe_register_loop_demo_agent(); + OpenclaWP_Agent_Registrar::maybe_register_site_introspection_agent(); + OpenclaWP_Agent_Registrar::maybe_register_workflow_drafter_agent(); + OpenclaWP_Agent_Registrar::maybe_register_example_agent(); + OpenclaWP_Agent_Registrar::maybe_register_coordinator_demo_agent(); + } + + /** + * @dataProvider canonical_pairs + */ + public function test_payload_matches_snapshot( string $snapshot_name, string $agent_slug, string $channel, string $user_message, array $prior_messages ): void { + $agent = wp_get_agent( $agent_slug ); + $this->assertInstanceOf( WP_Agent::class, $agent, sprintf( 'Fixture agent "%s" must be registered before assembly.', $agent_slug ) ); + + $payload = PromptPayloadAssembler::assemble( $agent, $channel, $user_message, $prior_messages ); + $actual = $this->encode_snapshot( $payload ); + + $snapshot_path = __DIR__ . '/__snapshots__/' . $snapshot_name . '.json'; + + if ( '1' === getenv( 'UPDATE_SNAPSHOTS' ) ) { + if ( ! is_dir( dirname( $snapshot_path ) ) ) { + mkdir( dirname( $snapshot_path ), 0o755, true ); + } + file_put_contents( $snapshot_path, $actual ); + $this->assertTrue( true, sprintf( 'Updated snapshot %s', $snapshot_name ) ); + return; + } + + $this->assertFileExists( + $snapshot_path, + sprintf( + "Snapshot file is missing for %s. Generate it with:\n UPDATE_SNAPSHOTS=1 composer test:assembly\n", + $snapshot_name + ) + ); + + $expected = (string) file_get_contents( $snapshot_path ); + $this->assertSame( + $expected, + $actual, + sprintf( + "Payload for %s drifted from snapshot.\nIf the change is intentional, regenerate with:\n UPDATE_SNAPSHOTS=1 composer test:assembly\n", + $snapshot_name + ) + ); + } + + /** + * @return array + */ + public function canonical_pairs(): array { + return array( + 'loop-demo-chat' => array( + 'loop-demo--chat', + 'openclawp-loop-demo', + 'chat', + 'What time is it?', + array(), + ), + 'site-introspection-whatsapp' => array( + 'site-introspection--whatsapp', + 'openclawp-site-introspection', + 'whatsapp', + 'How many comments are awaiting moderation?', + array(), + ), + 'coordinator-chat' => array( + 'coordinator--chat', + 'openclawp-coordinator', + 'chat', + 'Summarise the last three posts published on this site.', + array(), + ), + 'workflow-drafter-chat' => array( + 'workflow-drafter--chat', + 'openclawp-workflow-drafter', + 'chat', + 'When a new comment is posted, classify it for spam and notify me if it is spam.', + array(), + ), + 'example-whatsapp' => array( + 'example--whatsapp', + 'openclawp-example', + 'whatsapp', + 'Hello', + array(), + ), + ); + } + + /** + * Pretty-printed, deterministic JSON. Sorted keys + UTF-8 unescaped + a + * trailing newline so editors don't reflow the file on save. + */ + private function encode_snapshot( array $payload ): string { + $normalized = $this->sort_keys_recursive( $payload ); + $json = json_encode( + $normalized, + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR + ); + // json_encode pretty-prints with 4-space indent; collapse to tabs to + // match the repo's PHP/JS style guide and keep diffs compact. + $json = preg_replace_callback( + '/^( {4,})/m', + static fn( array $m ): string => str_repeat( "\t", (int) ( strlen( $m[1] ) / 4 ) ), + $json + ); + return ( $json ?? '' ) . "\n"; + } + + /** + * Recursively sort associative keys (leaves list arrays in their natural + * order, since order is meaningful for messages + tool catalogs). + * + * @param mixed $value Value to normalize. + * @return mixed + */ + private function sort_keys_recursive( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + + $is_list = array_is_list( $value ); + $out = array(); + foreach ( $value as $k => $v ) { + $out[ $k ] = $this->sort_keys_recursive( $v ); + } + if ( ! $is_list ) { + ksort( $out ); + } + return $out; + } + + /** + * Register the abilities referenced by the canonical demo agents. These + * mirror the real registrations in `OpenclaWP_Abilities` and + * `OpenclaWP_Site_Abilities` — descriptions + schemas only, no callbacks. + */ + private function register_canonical_abilities(): void { + $abilities = array( + 'openclawp/get-time' => array( + 'description' => 'Returns the current server time in ISO 8601 (UTC). Call this whenever the user asks for the time, the current date, or how long ago something happened.', + 'input_schema' => array( 'type' => 'object' ), + ), + 'openclawp/get-recent-posts' => array( + 'description' => 'Returns up to 5 most recent published posts (title + permalink + excerpt).', + 'input_schema' => array( + 'type' => 'object', + 'properties' => array( + 'limit' => array( 'type' => 'integer', 'minimum' => 1, 'maximum' => 10 ), + ), + ), + ), + 'openclawp/count-comments' => array( + 'description' => 'Returns counts of comments by status (approved, pending, spam, trash).', + 'input_schema' => array( 'type' => 'object' ), + ), + 'openclawp/get-active-plugins' => array( + 'description' => 'Lists currently active plugins on this site (name + version).', + 'input_schema' => array( 'type' => 'object' ), + ), + 'openclawp/get-current-user' => array( + 'description' => 'Returns the logged-in WordPress user (id, login, display name, roles).', + 'input_schema' => array( 'type' => 'object' ), + ), + ); + foreach ( $abilities as $name => $args ) { + wp_register_ability( $name, $args ); + } + } + +} diff --git a/tests/integration/prompt-assembly/PromptPayloadAssembler.php b/tests/integration/prompt-assembly/PromptPayloadAssembler.php new file mode 100644 index 0000000..e3f845f --- /dev/null +++ b/tests/integration/prompt-assembly/PromptPayloadAssembler.php @@ -0,0 +1,177 @@ + Deterministic JSON-serializable payload. + */ + public static function assemble( WP_Agent $agent, string $channel, string $user_message, array $prior_messages = array() ): array { + $tools = OpenclaWP_Tools_Resolver::for_agent( $agent ); + + // Mirror Runner::run_turn(): append the user's latest message to the + // transcript before the turn runner builds the provider payload. + $messages = $prior_messages; + $messages[] = array( + 'role' => 'user', + 'content' => $user_message, + ); + + $config = $agent->get_default_config(); + + // Tool declarations from the resolver are already in canonical's + // shape (executor=client, scope=run). Drop executor/scope from the + // snapshot — they're constants and only add noise. + $tool_catalog = array(); + foreach ( $tools['declarations'] as $name => $decl ) { + $tool_catalog[] = array( + 'name' => (string) ( $decl['name'] ?? $name ), + 'source' => (string) ( $decl['source'] ?? 'openclawp' ), + 'description' => (string) ( $decl['description'] ?? '' ), + 'parameters' => is_array( $decl['parameters'] ?? null ) ? $decl['parameters'] : array(), + ); + } + + return array( + 'agent_slug' => $agent->get_slug(), + 'channel' => $channel, + 'system_instruction' => $agent->get_description(), + 'model_preference' => self::resolve_model_preference( $config ), + 'messages' => self::normalize_messages( $messages ), + 'tool_catalog' => $tool_catalog, + 'delegate_targets' => $tools['delegate_targets'], + 'runtime_context' => self::client_context_for_channel( $channel ), + 'max_turns' => (int) ( $config['max_turns'] ?? 5 ), + ); + } + + /** + * Mirror of `OpenclaWP_Runner::resolve_model_preference()`. Re-implemented + * here so test snapshots are explicit about the rule, not implicit via a + * private method. + * + * @param array $config Agent default_config. + * @return array{provider:string,model:string}|array{model:string}|null + */ + private static function resolve_model_preference( array $config ) { + $model = isset( $config['model'] ) && is_string( $config['model'] ) ? trim( $config['model'] ) : ''; + $provider = isset( $config['provider'] ) && is_string( $config['provider'] ) ? trim( $config['provider'] ) : ''; + + if ( '' === $model || 'auto' === $model ) { + return null; + } + + if ( '' !== $provider && 'auto' !== $provider ) { + return array( + 'provider' => $provider, + 'model' => $model, + ); + } + + return array( 'model' => $model ); + } + + /** + * Canonical client_context map per channel. Matches what + * OpenclaWP_Whatsapp passes and what the chat block forwards. + * + * @return array + */ + private static function client_context_for_channel( string $channel ): array { + switch ( $channel ) { + case 'whatsapp': + return array( + 'client_context' => array( + 'source' => 'channel', + 'connector_id' => 'whatsapp', + 'client_name' => 'whatsapp', + 'external_provider' => 'whatsapp', + 'external_conversation_id' => '', + 'external_message_id' => '', + 'sender_id' => '', + 'room_kind' => 'dm', + ), + ); + case 'telegram': + return array( + 'client_context' => array( + 'source' => 'channel', + 'connector_id' => 'telegram', + 'client_name' => 'telegram', + 'external_provider' => 'telegram', + 'external_conversation_id' => '', + 'external_message_id' => '', + 'sender_id' => '', + 'room_kind' => 'dm', + ), + ); + case 'mcp': + return array( + 'client_context' => array( + 'source' => 'mcp', + 'connector_id' => 'mcp', + 'client_name' => 'mcp', + ), + ); + case 'chat': + default: + return array( + 'client_context' => array( + 'source' => 'block', + 'connector_id' => 'chat-block', + 'client_name' => 'chat-block', + ), + ); + } + } + + /** + * Strip non-string content variations so snapshots stay byte-stable. + * + * @param array> $messages Transcript. + * @return array + */ + private static function normalize_messages( array $messages ): array { + $out = array(); + foreach ( $messages as $message ) { + $out[] = array( + 'role' => (string) ( $message['role'] ?? '' ), + 'content' => (string) ( $message['content'] ?? '' ), + ); + } + return $out; + } +} diff --git a/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json new file mode 100644 index 0000000..3d3c7b3 --- /dev/null +++ b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json @@ -0,0 +1,62 @@ +{ + "agent_slug": "openclawp-coordinator", + "channel": "chat", + "delegate_targets": { + "delegate-to-openclawp-loop-demo": "openclawp-loop-demo", + "delegate-to-openclawp-site-introspection": "openclawp-site-introspection" + }, + "max_turns": 6, + "messages": [ + { + "content": "Summarise the last three posts published on this site.", + "role": "user" + } + ], + "model_preference": { + "model": "claude-haiku-4-5" + }, + "runtime_context": { + "client_context": { + "client_name": "chat-block", + "connector_id": "chat-block", + "source": "block" + } + }, + "system_instruction": "You are a coordinator. You have two subagents at your disposal — a site-introspection agent (read-only access to recent posts, comment counts, active plugins, current user) and a loop demo agent (the current time). Decide which subagent best fits the user's question, call the matching delegate-to-* tool with a focused prompt for that subagent, then summarise its reply. Never answer factual questions yourself — always delegate. If neither subagent fits, say so.", + "tool_catalog": [ + { + "description": "Delegate to subagent openclaWP Site Introspection. Use when the request is in this subagent's scope. Subagent description: You are a helpful assistant that answers questions about this WordPress site. You have read-only access to four tools: openclawp__get-recent-posts (recent published posts), openclawp__count-comments (comment moderation totals), openclawp__get-active-plugins (currently active plugins), and openclawp__get-current-user (the human you are talking to). Always call the relevant tool before answering a factual question — never guess. Quote tool output values directly. Be concise.", + "name": "delegate-to-openclawp-site-introspection", + "parameters": { + "properties": { + "prompt": { + "description": "The instruction to send to the subagent. The subagent receives only this text — include all the context it needs.", + "type": "string" + } + }, + "required": [ + "prompt" + ], + "type": "object" + }, + "source": "openclawp" + }, + { + "description": "Delegate to subagent openclaWP Loop Demo. Use when the request is in this subagent's scope. Subagent description: You are a precise assistant. You have access to one tool: openclawp__get-time, which returns the current time. When the user asks for the time, the current date, or anything time-related, you MUST call openclawp__get-time first and use its result in your reply. Never guess the time.", + "name": "delegate-to-openclawp-loop-demo", + "parameters": { + "properties": { + "prompt": { + "description": "The instruction to send to the subagent. The subagent receives only this text — include all the context it needs.", + "type": "string" + } + }, + "required": [ + "prompt" + ], + "type": "object" + }, + "source": "openclawp" + } + ] +} diff --git a/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json new file mode 100644 index 0000000..6d0b69c --- /dev/null +++ b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json @@ -0,0 +1,29 @@ +{ + "agent_slug": "openclawp-example", + "channel": "whatsapp", + "delegate_targets": [], + "max_turns": 5, + "messages": [ + { + "content": "Hello", + "role": "user" + } + ], + "model_preference": { + "model": "claude-haiku-4-5" + }, + "runtime_context": { + "client_context": { + "client_name": "whatsapp", + "connector_id": "whatsapp", + "external_conversation_id": "", + "external_message_id": "", + "external_provider": "whatsapp", + "room_kind": "dm", + "sender_id": "", + "source": "channel" + } + }, + "system_instruction": "Bundled example agent for smoke-testing openclaWP. Opt in via the openclawp_register_example_agent filter.", + "tool_catalog": [] +} diff --git a/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json new file mode 100644 index 0000000..5e6386e --- /dev/null +++ b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json @@ -0,0 +1,33 @@ +{ + "agent_slug": "openclawp-loop-demo", + "channel": "chat", + "delegate_targets": [], + "max_turns": 5, + "messages": [ + { + "content": "What time is it?", + "role": "user" + } + ], + "model_preference": { + "model": "claude-haiku-4-5" + }, + "runtime_context": { + "client_context": { + "client_name": "chat-block", + "connector_id": "chat-block", + "source": "block" + } + }, + "system_instruction": "You are a precise assistant. You have access to one tool: openclawp__get-time, which returns the current time. When the user asks for the time, the current date, or anything time-related, you MUST call openclawp__get-time first and use its result in your reply. Never guess the time.", + "tool_catalog": [ + { + "description": "Returns the current server time in ISO 8601 (UTC). Call this whenever the user asks for the time, the current date, or how long ago something happened.", + "name": "openclawp__get-time", + "parameters": { + "type": "object" + }, + "source": "openclawp" + } + ] +} diff --git a/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json new file mode 100644 index 0000000..238598b --- /dev/null +++ b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json @@ -0,0 +1,69 @@ +{ + "agent_slug": "openclawp-site-introspection", + "channel": "whatsapp", + "delegate_targets": [], + "max_turns": 6, + "messages": [ + { + "content": "How many comments are awaiting moderation?", + "role": "user" + } + ], + "model_preference": { + "model": "claude-haiku-4-5" + }, + "runtime_context": { + "client_context": { + "client_name": "whatsapp", + "connector_id": "whatsapp", + "external_conversation_id": "", + "external_message_id": "", + "external_provider": "whatsapp", + "room_kind": "dm", + "sender_id": "", + "source": "channel" + } + }, + "system_instruction": "You are a helpful assistant that answers questions about this WordPress site. You have read-only access to four tools: openclawp__get-recent-posts (recent published posts), openclawp__count-comments (comment moderation totals), openclawp__get-active-plugins (currently active plugins), and openclawp__get-current-user (the human you are talking to). Always call the relevant tool before answering a factual question — never guess. Quote tool output values directly. Be concise.", + "tool_catalog": [ + { + "description": "Returns up to 5 most recent published posts (title + permalink + excerpt).", + "name": "openclawp__get-recent-posts", + "parameters": { + "properties": { + "limit": { + "maximum": 10, + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + }, + "source": "openclawp" + }, + { + "description": "Returns counts of comments by status (approved, pending, spam, trash).", + "name": "openclawp__count-comments", + "parameters": { + "type": "object" + }, + "source": "openclawp" + }, + { + "description": "Lists currently active plugins on this site (name + version).", + "name": "openclawp__get-active-plugins", + "parameters": { + "type": "object" + }, + "source": "openclawp" + }, + { + "description": "Returns the logged-in WordPress user (id, login, display name, roles).", + "name": "openclawp__get-current-user", + "parameters": { + "type": "object" + }, + "source": "openclawp" + } + ] +} diff --git a/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json new file mode 100644 index 0000000..1669017 --- /dev/null +++ b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json @@ -0,0 +1,24 @@ +{ + "agent_slug": "openclawp-workflow-drafter", + "channel": "chat", + "delegate_targets": [], + "max_turns": 5, + "messages": [ + { + "content": "When a new comment is posted, classify it for spam and notify me if it is spam.", + "role": "user" + } + ], + "model_preference": { + "model": "claude-haiku-4-5" + }, + "runtime_context": { + "client_context": { + "client_name": "chat-block", + "connector_id": "chat-block", + "source": "block" + } + }, + "system_instruction": "You are a WordPress workflow author. Your job is to translate a one-paragraph description into a valid workflow spec for the openclaWP / agents-api Workflows substrate.\n\nSpec contract (JSON shape):\n{\n \"id\": \"/\",\n \"version\": \"1.0.0\",\n \"inputs\": { \"\": { \"type\": \"string\"|\"integer\"|\"boolean\", \"required\": true|false, \"description\": \"...\" } },\n \"steps\": [\n { \"id\": \"\", \"type\": \"ability\", \"ability\": \"\", \"args\": { ... } },\n { \"id\": \"\", \"type\": \"agent\", \"agent\": \"\", \"message\": \"...prompt for the LLM...\" }\n ],\n \"triggers\": [\n { \"type\": \"on_demand\" },\n { \"type\": \"wp_action\", \"hook\": \"\" },\n { \"type\": \"cron\", \"interval\": }\n ],\n \"meta\": { \"source_plugin\": \"openclawp/openclawp.php\", \"source_type\": \"user-drafted\" }\n}\n\nStep types:\n- `ability` — invokes a deterministic Abilities API ability. Use for read/write operations against WordPress, services, or external systems.\n- `agent` — calls an LLM via agents/chat. Use for reasoning, classification, summarization, decision-making.\n\nBindings (template syntax inside `args` / `message`):\n- `${inputs.}` — pulls from the workflow input\n- `${steps..output.}` — pulls from a previous step's output\n\nRules:\n1. Always include at least one step.\n2. Always include a `meta` object with `source_plugin` and `source_type` keys.\n3. If the user asks \"every time X happens\" use a `wp_action` trigger; \"every N minutes/hours\" → `cron`; otherwise `on_demand`.\n4. Prefer ability steps over agent steps when the operation is deterministic. Agent steps are for reasoning, not data fetching.\n5. Do not invent ability or agent slugs. Use the lists the caller provides as runtime context. If no fit, leave a placeholder slug (e.g. `my-plugin/my-ability`) and call it out in the explanation.\n\nWorked example. User says: *When a new comment is posted, classify it for spam and notify me if it's spam.*\n\n```json\n{\n \"id\": \"demo/spam-classify\",\n \"version\": \"1.0.0\",\n \"inputs\": { \"comment_id\": { \"type\": \"integer\", \"required\": true } },\n \"steps\": [\n {\n \"id\": \"classify\",\n \"type\": \"agent\",\n \"agent\": \"openclawp-site-introspection\",\n \"message\": \"Decide whether comment ${inputs.comment_id} is spam. Return JSON {\\\"is_spam\\\": true|false, \\\"reason\\\": \\\"...\\\"}.\"\n }\n ],\n \"triggers\": [\n { \"type\": \"wp_action\", \"hook\": \"comment_post\" }\n ],\n \"meta\": { \"source_plugin\": \"openclawp/openclawp.php\", \"source_type\": \"user-drafted\" }\n}\n```\n\nOutput format: respond with **the JSON spec inside a single ```json code fence**, then a short (one-paragraph) plain-English explanation **outside** the fence. Do not include any other prose before the code fence.", + "tool_catalog": [] +} diff --git a/tests/integration/prompt-assembly/bootstrap.php b/tests/integration/prompt-assembly/bootstrap.php new file mode 100644 index 0000000..373728f --- /dev/null +++ b/tests/integration/prompt-assembly/bootstrap.php @@ -0,0 +1,150 @@ + */ + private array $input_schema; + + public function __construct( string $name, string $description, array $input_schema ) { + $this->name = $name; + $this->description = $description; + $this->input_schema = $input_schema; + } + + public function get_name(): string { + return $this->name; + } + + public function get_description(): string { + return $this->description; + } + + /** + * @return array + */ + public function get_input_schema(): array { + return $this->input_schema; + } + } +} + +if ( ! function_exists( 'wp_register_ability' ) ) { + function wp_register_ability( string $name, array $args ) { + $ability = new Openclawp_Test_Stub_Ability( + $name, + (string) ( $args['description'] ?? '' ), + isset( $args['input_schema'] ) && is_array( $args['input_schema'] ) ? $args['input_schema'] : array( 'type' => 'object' ) + ); + $GLOBALS['openclawp_test_ability_registry'][ $name ] = $ability; + return $ability; + } +} +if ( ! function_exists( 'wp_get_ability' ) ) { + function wp_get_ability( string $name ) { + return $GLOBALS['openclawp_test_ability_registry'][ $name ] ?? null; + } +} +if ( ! function_exists( 'wp_has_ability' ) ) { + function wp_has_ability( string $name ): bool { + return isset( $GLOBALS['openclawp_test_ability_registry'][ $name ] ); + } +} + +if ( ! function_exists( 'wp_register_agent' ) ) { + function wp_register_agent( string $slug, array $args ) { + $agent = new WP_Agent( $slug, $args ); + $GLOBALS['openclawp_test_agent_registry'][ $agent->get_slug() ] = $agent; + return $agent; + } +} +if ( ! function_exists( 'wp_get_agent' ) ) { + function wp_get_agent( string $slug ) { + return $GLOBALS['openclawp_test_agent_registry'][ $slug ] ?? null; + } +} +if ( ! function_exists( 'wp_has_agent' ) ) { + function wp_has_agent( string $slug ): bool { + return isset( $GLOBALS['openclawp_test_agent_registry'][ $slug ] ); + } +} + +// Helpers normally provided by core that the registrar / abilities touch. +if ( ! function_exists( '__' ) ) { + function __( string $text, string $domain = 'default' ): string { + unset( $domain ); + return $text; + } +} +if ( ! function_exists( 'esc_html' ) ) { + function esc_html( string $text ): string { + return htmlspecialchars( $text, ENT_QUOTES, 'UTF-8' ); + } +} +if ( ! function_exists( 'sanitize_title' ) ) { + function sanitize_title( string $title ): string { + $title = strtolower( trim( $title ) ); + $title = preg_replace( '/[^a-z0-9-]+/', '-', $title ); + $title = trim( (string) $title, '-' ); + return (string) $title; + } +} +if ( ! function_exists( 'sanitize_file_name' ) ) { + function sanitize_file_name( string $name ): string { + return preg_replace( '/[^A-Za-z0-9._-]+/', '', $name ) ?? ''; + } +} +if ( ! function_exists( 'get_current_user_id' ) ) { + function get_current_user_id(): int { + return 0; + } +} diff --git a/tests/integration/prompt-assembly/stubs/function-declaration-stub.php b/tests/integration/prompt-assembly/stubs/function-declaration-stub.php new file mode 100644 index 0000000..dddbdde --- /dev/null +++ b/tests/integration/prompt-assembly/stubs/function-declaration-stub.php @@ -0,0 +1,23 @@ +