Distinguish diagnostics from runtime source (#1778)

Author: chubes4

.github/scripts/run-agent-task/prepare-agent-task-upload.mjs

@@ -15,7 +15,17 @@ const runtimeSourcePrefix = process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX ? resol
 const runtimeSourceRoots = [runtimeSourceRoot, runtimeSourcePrefix].filter(Boolean)
 const SOURCE_TREE = /(^|\/)(prepared-plugins|prepared-source-packages|source-package[^/]*)(\/|$)/i
 const SOURCE_FILE = /\.(?:php|phtml|js|mjs|cjs|jsx|ts|tsx)$/i
-const RUNTIME_SOURCE_CONTENT = /(?:Plugin Name:|WP_Agents_Registry|OpenAiProvider)/
+const PHP_OPENING_TAG = /<\?(?:php|=)(?:\s|$)/i
+const PHP_DECLARATION = /\b(?:namespace\s+\\?[A-Za-z_]\w*(?:\\[A-Za-z_]\w*)*|(?:abstract\s+|final\s+|readonly\s+)*(?:class|interface|trait|enum)\s+[A-Za-z_]\w*|function\s+&?\s*[A-Za-z_]\w*\s*\()/i
+const WORDPRESS_PLUGIN_HEADER = /\/\*[\s\S]{0,200}?\bPlugin Name\s*:/i
+
+// Diagnostics commonly name runtime classes. Reject only PHP-shaped source, even
+// when a source file has been disguised with a reviewer-safe extension.
+function containsRuntimeSourceContent(text) {
+  const hasPhpTag = PHP_OPENING_TAG.test(text)
+  const hasDeclaration = PHP_DECLARATION.test(text)
+  return (hasPhpTag && hasDeclaration) || (WORDPRESS_PLUGIN_HEADER.test(text) && (hasPhpTag || hasDeclaration))
+}
 
 function redact(value) {
   return secretValues.reduce((output, secret) => output.split(secret).join("[REDACTED]"), value)
@@ -74,7 +84,7 @@ async function stageTextFile(source, destination, options = {}) {
   if (!contents || contents.includes(0) || !isUtf8(contents)) return false
   const text = redact(options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8")))
   assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.")
-  if (RUNTIME_SOURCE_CONTENT.test(text)) throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.")
+  if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.")
   await mkdir(resolve(destination, ".."), { recursive: true })
   await writeFile(destination, text)
   return true
@@ -154,7 +164,7 @@ async function finalScan(directory) {
       const bytes = await readFile(path)
       const text = isUtf8(bytes) ? bytes.toString("utf8") : ""
       assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.")
-      if (RUNTIME_SOURCE_CONTENT.test(text)) throw new Error("Prepared runtime plugin source contents must never be persisted in artifact uploads.")
+      if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be persisted in artifact uploads.")
     } else throw new Error("Only regular files may be persisted in artifact uploads.")
   }
 }

fixtures/agent-task-upload-run-29307978522.json

@@ -0,0 +1,15 @@
+{
+  "run_id": "29307978522",
+  "workflow": "Build With WordPress Skills Agent",
+  "result": {
+    "schema": "wp-codebox/agent-task-workflow-result/v1",
+    "status": "failed",
+    "success": false,
+    "diagnostics": [
+      {
+        "message": "Unable to resolve WordPress\\OpenAiAiProvider\\Provider\\OpenAiProvider for the selected model.",
+        "stack": "Error: provider resolution failed\n    at WP_Agents_Registry::get_instance (runtime.php:42)\n    at executeAgentTask (runner.php:87)"
+      }
+    ]
+  }
+}

fixtures/agent-task-upload/runtime-source-disguised.json

@@ -0,0 +1,4 @@
+<?php
+namespace Runtime\Provider;
+
+final class OpenAiProvider {}

fixtures/agent-task-upload/runtime-source-disguised.txt

@@ -0,0 +1,5 @@
+<?php
+/*
+ * Plugin Name: Disguised Runtime Provider
+ */
+function runtime_provider_bootstrap() {}

tests/runtime-sources-materialization.test.ts

@@ -17,6 +17,10 @@ const uploadLayoutRegression = JSON.parse(await readFile(new URL("../fixtures/ag
 assert.equal(uploadLayoutRegression.run_id, "29306539573")
 assert.equal(uploadLayoutRegression.observed.upload_preparation, "failed-on-runtime-source")
 assert.match(uploadLayoutRegression.raw_layout.runtime_source, /prepared-plugins\/agents-api\/agents-api\.php$/)
+const diagnosticRegression = JSON.parse(await readFile(new URL("../fixtures/agent-task-upload-run-29307978522.json", import.meta.url), "utf8"))
+assert.equal(diagnosticRegression.run_id, "29307978522")
+assert.match(diagnosticRegression.result.diagnostics[0].message, /OpenAiProvider/)
+assert.match(diagnosticRegression.result.diagnostics[0].stack, /WP_Agents_Registry/)
 const hostedPathRegression = JSON.parse(await readFile(new URL("../fixtures/agent-task-runtime-paths-run-29305012941.json", import.meta.url), "utf8"))
 for (const result of [hostedPathRegression.success, hostedPathRegression.failure]) {
   const sanitized = sanitizeRuntimeSourceValue(result, hostedPathRegression.runtime_root)
@@ -151,6 +155,15 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => {
   assert.doesNotMatch(stagedInput, /component_contracts|source_package_root/)
   const stagedResult = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8")
   assert.doesNotMatch(stagedResult, /private-runtime-source/)
+  await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), `${JSON.stringify(diagnosticRegression.result, null, 2)}\n`)
+  await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } })
+  const stagedDiagnostic = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8")
+  assert.match(stagedDiagnostic, /OpenAiProvider/)
+  assert.match(stagedDiagnostic, /WP_Agents_Registry/)
+  await writeFile(join(artifacts, "safe.json"), "Diagnostic snippet: <?php OpenAiProvider function was unavailable in WP_Agents_Registry.")
+  await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "safe.json" } }] }))
+  await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } })
+  assert.match(await readFile(join(upload, ".codebox", "agent-task-artifacts", "safe.json"), "utf8"), /OpenAiProvider/)
   await writeFile(join(artifacts, "leak.json"), `runtime log ${privateRoot}/source.php`)
   await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ status: "failed", success: false, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "leak.json" } }] }))
   await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } })
@@ -167,7 +180,15 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => {
   assert.doesNotMatch(exclusionManifest, /prepared-plugins|agents-api|private-runtime-source/)
   await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "prepared-plugins/agents-api/agents-api.php" } }] }))
   await assert.rejects(execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }), /Declared reviewer artifacts/)
+  assert.ok(await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8"), "Declared artifact rejection preserves the normalized control result")
   await rm(join(artifacts, "prepared-plugins"), { recursive: true, force: true })
+  for (const path of ["runtime-source-disguised.json", "runtime-source-disguised.txt"]) {
+    await writeFile(join(artifacts, path), await readFile(new URL(`../fixtures/agent-task-upload/${path}`, import.meta.url), "utf8"))
+    await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path } }] }))
+    await assert.rejects(execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }), /source contents/)
+    assert.ok(await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8"), "Disguised source rejection preserves the normalized control result")
+    await rm(join(artifacts, path))
+  }
   await mkdir(suffixedPrivateRoot, { recursive: true })
   await writeFile(join(artifacts, "suffixed-root-leak.json"), suffixedPrivateRoot)
   await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "suffixed-root-leak.json" } }] }))