Evals (Layer 1): payload snapshot tests for prompt regressions (closes #46 Layer 1)

.github/workflows/tests.yml

@@ -82,6 +82,14 @@ jobs:
       - name: Run PHPUnit (unit suite)
         run: vendor/bin/phpunit --testsuite unit
 
+      - name: Run prompt-assembly snapshot suite
+        # Fails the build if the assembled provider payload (system prompt,
+        # tool catalog, model preference, runtime context) drifts from the
+        # committed snapshot for any canonical (agent, channel) pair.
+        # Update flow: `UPDATE_SNAPSHOTS=1 composer test:assembly` — the
+        # regenerated snapshot then goes through PR review.
+        run: composer test:assembly
+
       - name: Run PHPUnit (integration suite)
         if: hashFiles('tests/integration/**') != ''
         run: vendor/bin/phpunit --testsuite integration

composer.json

@@ -39,6 +39,7 @@
   "scripts": {
     "test": "phpunit",
     "test:smoke": "php tests/smoke.php",
+    "test:assembly": "phpunit --testsuite assembly",
     "lint:php": "phpcs",
     "lint:php:fix": "phpcbf",
     "analyse": "phpstan analyse"

docs/dev/evals.md

@@ -0,0 +1,92 @@
+# Prompt evals
+
+openclaWP guards against prompt regressions in two complementary layers. Layer 1
+is shipped today; Layer 2 is queued. Tracking issue:
+[#46](https://github.com/lezama/openclawp/issues/46).
+
+## Layer 1 — payload snapshot tests (shipping)
+
+Goal: catch unintentional drift in the provider payload (system prompt, message
+list, tool catalog, model preference) at zero token cost.
+
+For each canonical `(agent, channel)` pair the suite freezes a user message,
+asks the production code to build the request payload openclaWP would send to
+the provider, and diffs that payload against a committed JSON snapshot under
+`tests/integration/prompt-assembly/__snapshots__/`.
+
+Pairs currently covered:
+
+| Snapshot                            | Agent                            | Channel    |
+|-------------------------------------|----------------------------------|------------|
+| `loop-demo--chat.json`              | `openclawp-loop-demo`            | `chat`     |
+| `site-introspection--whatsapp.json` | `openclawp-site-introspection`   | `whatsapp` |
+| `coordinator--chat.json`            | `openclawp-coordinator`          | `chat`     |
+| `workflow-drafter--chat.json`       | `openclawp-workflow-drafter`     | `chat`     |
+| `example--whatsapp.json`            | `openclawp-example`              | `whatsapp` |
+
+### Run
+
+```bash
+composer test:assembly
+```
+
+Runs on every CI build inside the existing `phpunit` job (`.github/workflows/tests.yml`).
+Adds <1s to the test cycle — no model calls, no network.
+
+### Failure mode
+
+A failing snapshot prints a unified diff of expected vs actual JSON and points
+at the regenerate command. Example:
+
+```
+Failed asserting that two strings are identical.
+--- Expected
++++ Actual
+@@ @@
+-	"system_instruction": "You are a precise assistant. …",
++	"system_instruction": "You are a regressed assistant. …",
+```
+
+### Updating snapshots (intentional changes)
+
+When you change a system prompt, tool description, or default config on
+purpose, regenerate the affected snapshots and commit the diff:
+
+```bash
+UPDATE_SNAPSHOTS=1 composer test:assembly
+git add tests/integration/prompt-assembly/__snapshots__
+```
+
+The PR reviewer reads the diff to confirm the change is intentional. Treat
+every snapshot churn the same way you'd treat a copy edit on a customer-facing
+string.
+
+### Adding a new pair
+
+Add a row to `canonical_pairs()` in
+`tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php`, then run:
+
+```bash
+UPDATE_SNAPSHOTS=1 composer test:assembly
+```
+
+The first run creates the snapshot. Subsequent runs diff against it.
+
+### Implementation notes
+
+- The assembler lives in
+  `tests/integration/prompt-assembly/PromptPayloadAssembler.php`. It mirrors
+  `OpenclaWP_Runner::build_turn_runner()` minus the provider call — same
+  agent description, same transcript, same `OpenclaWP_Tools_Resolver` output,
+  same model-preference resolver.
+- JSON is pretty-printed with sorted keys (recursive on assoc arrays, preserving
+  list order). Trailing newline. No machine-specific paths. Tabs for indent so
+  the files match the repo's PHP/JS style.
+- Zero new composer deps. Plain PHPUnit + `file_put_contents()` for updates.
+
+## Layer 2 — black-box conversation evals (deferred)
+
+YAML-defined eval suite covering 10 seed conversations across chat block,
+WhatsApp, Telegram, and MCP, executed by promptfoo's HTTP provider against a
+booted wp-env. Tracked in
+[#49](https://github.com/lezama/openclawp/issues/49).

phpunit.xml.dist

@@ -10,5 +10,14 @@
 		<testsuite name="unit">
 			<directory>tests/unit</directory>
 		</testsuite>
+		<testsuite name="integration">
+			<directory>tests/integration</directory>
+			<!-- The assembly suite has its own composer script + CI step. Exclude
+				 it from the broader integration suite so it doesn't run twice. -->
+			<exclude>tests/integration/prompt-assembly</exclude>
+		</testsuite>
+		<testsuite name="assembly">
+			<directory>tests/integration/prompt-assembly</directory>
+		</testsuite>
 	</testsuites>
 </phpunit>

tests/integration/prompt-assembly/PromptAssemblySnapshotTest.php

@@ -0,0 +1,233 @@
+<?php
+/**
+ * Golden-trace snapshot tests for the assembled provider payload.
+ *
+ * For each canonical (agent, channel) pair we freeze a user message, build
+ * the payload openclaWP would send to the provider, and diff it against
+ * a JSON snapshot under `__snapshots__/`. Any change to the system prompt,
+ * tool descriptions, parameter shape, or default config will surface as a
+ * failing test on the next CI run.
+ *
+ * Update flow (intentional changes):
+ *
+ *     UPDATE_SNAPSHOTS=1 composer test:assembly
+ *
+ * The regenerated snapshot is then reviewed in the PR diff.
+ *
+ * @package OpenclaWP\Tests\Integration\PromptAssembly
+ */
+
+declare( strict_types=1 );
+
+namespace OpenclaWP\Tests\Integration\PromptAssembly;
+
+require_once __DIR__ . '/bootstrap.php';
+
+use OpenclaWP_Agent_Registrar;
+use PHPUnit\Framework\TestCase;
+use WP_Agent;
+
+/**
+ * @covers \OpenclaWP_Tools_Resolver
+ * @covers \OpenclaWP_Agent_Registrar
+ */
+final class PromptAssemblySnapshotTest extends TestCase {
+
+	/**
+	 * Pre-populated registries reset before each test so the registrar's
+	 * idempotency guards (`wp_has_agent`) re-run cleanly. Opt-in filters that
+	 * normally gate the demo agents are flipped on here — that way the
+	 * snapshot exercises the production registrar's real registration args
+	 * (descriptions, default_config, subagents) instead of a duplicated
+	 * copy.
+	 */
+	protected function setUp(): void {
+		$GLOBALS['openclawp_test_ability_registry'] = array();
+		$GLOBALS['openclawp_test_agent_registry']   = array();
+
+		$true_filter = static fn( $value ) => true;
+		$GLOBALS['openclawp_test_filters'] = array(
+			'openclawp_register_loop_demo'           => $true_filter,
+			'openclawp_register_site_introspection'  => $true_filter,
+			'openclawp_register_workflow_drafter'    => $true_filter,
+			'openclawp_register_coordinator_demo'    => $true_filter,
+			'openclawp_register_example_agent'       => $true_filter,
+		);
+
+		$this->register_canonical_abilities();
+
+		// Run the production registrar so the snapshot reflects the *real*
+		// agent args. Subagent-coordinator registers last; the registrar's
+		// own ordering (priority 20) is preserved here by call order.
+		OpenclaWP_Agent_Registrar::maybe_register_loop_demo_agent();
+		OpenclaWP_Agent_Registrar::maybe_register_site_introspection_agent();
+		OpenclaWP_Agent_Registrar::maybe_register_workflow_drafter_agent();
+		OpenclaWP_Agent_Registrar::maybe_register_example_agent();
+		OpenclaWP_Agent_Registrar::maybe_register_coordinator_demo_agent();
+	}
+
+	/**
+	 * @dataProvider canonical_pairs
+	 */
+	public function test_payload_matches_snapshot( string $snapshot_name, string $agent_slug, string $channel, string $user_message, array $prior_messages ): void {
+		$agent = wp_get_agent( $agent_slug );
+		$this->assertInstanceOf( WP_Agent::class, $agent, sprintf( 'Fixture agent "%s" must be registered before assembly.', $agent_slug ) );
+
+		$payload = PromptPayloadAssembler::assemble( $agent, $channel, $user_message, $prior_messages );
+		$actual  = $this->encode_snapshot( $payload );
+
+		$snapshot_path = __DIR__ . '/__snapshots__/' . $snapshot_name . '.json';
+
+		if ( '1' === getenv( 'UPDATE_SNAPSHOTS' ) ) {
+			if ( ! is_dir( dirname( $snapshot_path ) ) ) {
+				mkdir( dirname( $snapshot_path ), 0o755, true );
+			}
+			file_put_contents( $snapshot_path, $actual );
+			$this->assertTrue( true, sprintf( 'Updated snapshot %s', $snapshot_name ) );
+			return;
+		}
+
+		$this->assertFileExists(
+			$snapshot_path,
+			sprintf(
+				"Snapshot file is missing for %s. Generate it with:\n  UPDATE_SNAPSHOTS=1 composer test:assembly\n",
+				$snapshot_name
+			)
+		);
+
+		$expected = (string) file_get_contents( $snapshot_path );
+		$this->assertSame(
+			$expected,
+			$actual,
+			sprintf(
+				"Payload for %s drifted from snapshot.\nIf the change is intentional, regenerate with:\n  UPDATE_SNAPSHOTS=1 composer test:assembly\n",
+				$snapshot_name
+			)
+		);
+	}
+
+	/**
+	 * @return array<string, array{0:string,1:string,2:string,3:string,4:array}>
+	 */
+	public function canonical_pairs(): array {
+		return array(
+			'loop-demo-chat'             => array(
+				'loop-demo--chat',
+				'openclawp-loop-demo',
+				'chat',
+				'What time is it?',
+				array(),
+			),
+			'site-introspection-whatsapp' => array(
+				'site-introspection--whatsapp',
+				'openclawp-site-introspection',
+				'whatsapp',
+				'How many comments are awaiting moderation?',
+				array(),
+			),
+			'coordinator-chat'           => array(
+				'coordinator--chat',
+				'openclawp-coordinator',
+				'chat',
+				'Summarise the last three posts published on this site.',
+				array(),
+			),
+			'workflow-drafter-chat'      => array(
+				'workflow-drafter--chat',
+				'openclawp-workflow-drafter',
+				'chat',
+				'When a new comment is posted, classify it for spam and notify me if it is spam.',
+				array(),
+			),
+			'example-whatsapp'           => array(
+				'example--whatsapp',
+				'openclawp-example',
+				'whatsapp',
+				'Hello',
+				array(),
+			),
+		);
+	}
+
+	/**
+	 * Pretty-printed, deterministic JSON. Sorted keys + UTF-8 unescaped + a
+	 * trailing newline so editors don't reflow the file on save.
+	 */
+	private function encode_snapshot( array $payload ): string {
+		$normalized = $this->sort_keys_recursive( $payload );
+		$json       = json_encode(
+			$normalized,
+			JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR
+		);
+		// json_encode pretty-prints with 4-space indent; collapse to tabs to
+		// match the repo's PHP/JS style guide and keep diffs compact.
+		$json = preg_replace_callback(
+			'/^( {4,})/m',
+			static fn( array $m ): string => str_repeat( "\t", (int) ( strlen( $m[1] ) / 4 ) ),
+			$json
+		);
+		return ( $json ?? '' ) . "\n";
+	}
+
+	/**
+	 * Recursively sort associative keys (leaves list arrays in their natural
+	 * order, since order is meaningful for messages + tool catalogs).
+	 *
+	 * @param mixed $value Value to normalize.
+	 * @return mixed
+	 */
+	private function sort_keys_recursive( $value ) {
+		if ( ! is_array( $value ) ) {
+			return $value;
+		}
+
+		$is_list = array_is_list( $value );
+		$out     = array();
+		foreach ( $value as $k => $v ) {
+			$out[ $k ] = $this->sort_keys_recursive( $v );
+		}
+		if ( ! $is_list ) {
+			ksort( $out );
+		}
+		return $out;
+	}
+
+	/**
+	 * Register the abilities referenced by the canonical demo agents. These
+	 * mirror the real registrations in `OpenclaWP_Abilities` and
+	 * `OpenclaWP_Site_Abilities` — descriptions + schemas only, no callbacks.
+	 */
+	private function register_canonical_abilities(): void {
+		$abilities = array(
+			'openclawp/get-time' => array(
+				'description'  => 'Returns the current server time in ISO 8601 (UTC). Call this whenever the user asks for the time, the current date, or how long ago something happened.',
+				'input_schema' => array( 'type' => 'object' ),
+			),
+			'openclawp/get-recent-posts' => array(
+				'description'  => 'Returns up to 5 most recent published posts (title + permalink + excerpt).',
+				'input_schema' => array(
+					'type'       => 'object',
+					'properties' => array(
+						'limit' => array( 'type' => 'integer', 'minimum' => 1, 'maximum' => 10 ),
+					),
+				),
+			),
+			'openclawp/count-comments' => array(
+				'description'  => 'Returns counts of comments by status (approved, pending, spam, trash).',
+				'input_schema' => array( 'type' => 'object' ),
+			),
+			'openclawp/get-active-plugins' => array(
+				'description'  => 'Lists currently active plugins on this site (name + version).',
+				'input_schema' => array( 'type' => 'object' ),
+			),
+			'openclawp/get-current-user' => array(
+				'description'  => 'Returns the logged-in WordPress user (id, login, display name, roles).',
+				'input_schema' => array( 'type' => 'object' ),
+			),
+		);
+		foreach ( $abilities as $name => $args ) {
+			wp_register_ability( $name, $args );
+		}
+	}
+
+}