chore(cli): implement test harness (#38914)

pavelfeldman · web-flow · commit 95a728a03707 · 2026-01-22T20:57:43.000-08:00
diff --git a/packages/playwright/src/mcp/browser/response.ts b/packages/playwright/src/mcp/browser/response.ts
@@ -72,12 +72,12 @@ export class Response {
     // What can go into a file goes into a file in outputMode === file.
     if (this._context.config.outputMode === 'file') {
       if (!result.suggestedFilename)
-        result.suggestedFilename = dateAsFileName(result.ext ?? (result.text ? 'txt' : 'bin'));
+        result.suggestedFilename = dateAsFileName(result.ext ?? (result.text !== undefined ? 'txt' : 'bin'));
     }
 
     const entry: Result = { text: result.text, data: result.data, title: result.title };
     if (result.suggestedFilename)
-      entry.filename = await this._context.outputFile(result.suggestedFilename, { origin: 'llm', title: result.title ?? 'Saved result' });
+      entry.filename = await this._context.outputFile(result.suggestedFilename, { origin: 'llm', title: result.title || 'Saved result' });
 
     this._results.push(entry);
     return { fileName: entry.filename };
@@ -124,10 +124,11 @@ export class Response {
       const text = addSection('Result');
       for (const result of this._results) {
         if (result.filename) {
-          text.push(`- [${result.title}](${rootPath ? path.relative(rootPath, result.filename) : result.filename})`);
+          if (result.text !== undefined || result.data)
+            text.push(`- [${result.title}](${rootPath ? path.relative(rootPath, result.filename) : result.filename})`);
           if (result.data)
             await fs.promises.writeFile(result.filename, result.data);
-          else if (result.text)
+          else if (result.text !== undefined)
             await fs.promises.writeFile(result.filename, this._redactText(result.text));
         } else if (result.text) {
           text.push(result.text);
@@ -247,7 +248,7 @@ export function renderTabMarkdown(tab: TabHeader): string[] {
 
 export function renderTabsMarkdown(tabs: TabHeader[]): string[] {
   if (!tabs.length)
-    return ['No open tabs. Use the "browser_navigate" tool to navigate to a page first.'];
+    return ['No open tabs. Navigate to a URL to create one.'];
 
   const lines: string[] = [];
   for (let i = 0; i < tabs.length; i++) {
diff --git a/packages/playwright/src/mcp/browser/tools/common.ts b/packages/playwright/src/mcp/browser/tools/common.ts
@@ -52,10 +52,7 @@ const resize = defineTabTool({
 
   handle: async (tab, params, response) => {
     response.addCode(`await page.setViewportSize({ width: ${params.width}, height: ${params.height} });`);
-
-    await tab.waitForCompletion(async () => {
-      await tab.page.setViewportSize({ width: params.width, height: params.height });
-    });
+    await tab.page.setViewportSize({ width: params.width, height: params.height });
   },
 });
 
diff --git a/packages/playwright/src/mcp/browser/tools/console.ts b/packages/playwright/src/mcp/browser/tools/console.ts
@@ -32,7 +32,7 @@ const console = defineTabTool({
   handle: async (tab, params, response) => {
     const messages = await tab.consoleMessages(params.level);
     const text = messages.map(message => message.toString()).join('\n');
-    await response.addResult({ text, suggestedFilename: params.filename });
+    await response.addResult({ text, suggestedFilename: params.filename, title: 'Console' });
   },
 });
 
diff --git a/packages/playwright/src/mcp/browser/tools/dialogs.ts b/packages/playwright/src/mcp/browser/tools/dialogs.ts
@@ -32,8 +32,6 @@ export const handleDialog = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    response.setIncludeSnapshot();
-
     const dialogState = tab.modalStates().find(state => state.type === 'dialog');
     if (!dialogState)
       throw new Error('No dialog visible');
diff --git a/packages/playwright/src/mcp/browser/tools/evaluate.ts b/packages/playwright/src/mcp/browser/tools/evaluate.ts
@@ -25,7 +25,6 @@ const evaluateSchema = z.object({
   function: z.string().describe('() => { /* code */ } or (element) => { /* code */ } when element is provided'),
   element: z.string().optional().describe('Human-readable element description used to obtain permission to interact with the element'),
   ref: z.string().optional().describe('Exact target element reference from the page snapshot'),
-  filename: z.string().optional().describe('Filename to save the result to. If not provided, result is returned as JSON string.'),
 });
 
 const evaluate = defineTabTool({
@@ -39,11 +38,9 @@ const evaluate = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    response.setIncludeSnapshot();
-
     let locator: Awaited<ReturnType<Tab['refLocator']>> | undefined;
-    if (params.ref && params.element) {
-      locator = await tab.refLocator({ ref: params.ref, element: params.element });
+    if (params.ref) {
+      locator = await tab.refLocator({ ref: params.ref, element: params.element || 'element' });
       response.addCode(`await page.${locator.resolved}.evaluate(${escapeWithQuotes(params.function)});`);
     } else {
       response.addCode(`await page.evaluate(${escapeWithQuotes(params.function)});`);
@@ -53,7 +50,7 @@ const evaluate = defineTabTool({
       const receiver = locator?.locator ?? tab.page;
       const result = await receiver._evaluateFunction(params.function);
       const text = JSON.stringify(result, null, 2) || 'undefined';
-      await response.addResult({ text, suggestedFilename: params.filename });
+      response.addTextResult(text);
     });
   },
 });
diff --git a/packages/playwright/src/mcp/browser/tools/network.ts b/packages/playwright/src/mcp/browser/tools/network.ts
@@ -42,7 +42,7 @@ const requests = defineTabTool({
       if (rendered)
         text.push(rendered);
     }
-    await response.addResult({ text: text.join('\n'), suggestedFilename: params.filename });
+    await response.addResult({ text: text.join('\n'), suggestedFilename: params.filename, title: 'Network' });
   },
 });
 
diff --git a/packages/playwright/src/mcp/browser/tools/runCode.ts b/packages/playwright/src/mcp/browser/tools/runCode.ts
@@ -23,7 +23,6 @@ import { defineTabTool } from './tool';
 
 const codeSchema = z.object({
   code: z.string().describe(`A JavaScript function containing Playwright code to execute. It will be invoked with a single argument, page, which you can use for any page interaction. For example: \`async (page) => { await page.getByRole('button', { name: 'Submit' }).click(); return await page.title(); }\``),
-  filename: z.string().optional().describe('Filename to save the result to. If not provided, result is returned as JSON string.'),
 });
 
 const runCode = defineTabTool({
@@ -37,7 +36,6 @@ const runCode = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    response.setIncludeSnapshot();
     response.addCode(`await (${params.code})(page);`);
     const __end__ = new ManualPromise<void>();
     const context = {
@@ -57,7 +55,7 @@ const runCode = defineTabTool({
       await vm.runInContext(snippet, context);
       const result = await __end__;
       if (typeof result === 'string')
-        await response.addResult({ text: result, suggestedFilename: params.filename });
+        response.addTextResult(result);
     });
   },
 });
diff --git a/packages/playwright/src/mcp/browser/tools/screenshot.ts b/packages/playwright/src/mcp/browser/tools/screenshot.ts
@@ -43,8 +43,6 @@ const screenshot = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    if (!!params.element !== !!params.ref)
-      throw new Error('Both element and ref must be provided or neither.');
     if (params.fullPage && params.ref)
       throw new Error('fullPage cannot be used with element screenshots.');
 
@@ -55,9 +53,8 @@ const screenshot = defineTabTool({
       scale: 'css',
       ...(params.fullPage !== undefined && { fullPage: params.fullPage })
     };
-    const isElementScreenshot = params.element && params.ref;
 
-    const screenshotTarget = isElementScreenshot ? params.element : (params.fullPage ? 'full page' : 'viewport');
+    const screenshotTarget = params.ref ? params.element || 'element' : (params.fullPage ? 'full page' : 'viewport');
     const ref = params.ref ? await tab.refLocator({ element: params.element || '', ref: params.ref }) : null;
 
     const data = ref ? await ref.locator.screenshot(options) : await tab.page.screenshot(options);
diff --git a/packages/playwright/src/mcp/browser/tools/snapshot.ts b/packages/playwright/src/mcp/browser/tools/snapshot.ts
@@ -191,8 +191,9 @@ const check = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    const { resolved } = await tab.refLocator(params);
+    const { locator, resolved } = await tab.refLocator(params);
     response.addCode(`await page.${resolved}.check();`);
+    await locator.check();
   },
 });
 
@@ -208,8 +209,9 @@ const uncheck = defineTabTool({
   },
 
   handle: async (tab, params, response) => {
-    const { resolved } = await tab.refLocator(params);
+    const { locator, resolved } = await tab.refLocator(params);
     response.addCode(`await page.${resolved}.uncheck();`);
+    await locator.uncheck();
   },
 });
 
diff --git a/packages/playwright/src/mcp/terminal/cli.ts b/packages/playwright/src/mcp/terminal/cli.ts
@@ -18,6 +18,7 @@
 /* eslint-disable no-restricted-properties */
 
 import { spawn } from 'child_process';
+
 import crypto from 'crypto';
 import fs from 'fs';
 import net from 'net';
@@ -26,6 +27,8 @@ import path from 'path';
 import { debug } from 'playwright-core/lib/utilsBundle';
 import { SocketConnection } from './socketConnection';
 
+import type { SpawnOptions } from 'child_process';
+
 const debugCli = debug('pw:cli');
 const packageJSON = require('../../../package.json');
 
@@ -158,7 +161,7 @@ class SessionManager {
   }
 
   private async _connect(sessionName: string): Promise<Session> {
-    const socketPath = this._daemonSocketPath(sessionName);
+    const socketPath = process.env.PLAYWRIGHT_DAEMON_SOCKET_PATH || this._daemonSocketPath(sessionName);
     debugCli(`Connecting to daemon at ${socketPath}`);
 
     const socketExists = await fs.promises.stat(socketPath)
@@ -176,11 +179,11 @@ class SessionManager {
       }
     }
 
-    const cliPath = path.join(__dirname, '../../../cli.js');
-    debugCli(`Will launch daemon process: ${cliPath}`);
+    if (process.env.PLAYWRIGHT_DAEMON_SOCKET_PATH)
+      throw new Error(`Socket path ${socketPath} does not exist`);
 
     const userDataDir = path.resolve(daemonSocketDir, `${sessionName}-user-data`);
-    const child = spawn(process.execPath, [cliPath, 'run-mcp-server', `--daemon=${socketPath}`, `--user-data-dir=${userDataDir}`], {
+    const child = spawnDaemon(socketPath, userDataDir, {
       detached: true,
       stdio: 'ignore',
       cwd: process.cwd(), // Will be used as root.
@@ -298,6 +301,12 @@ const daemonSocketDir = (() => {
   return path.join(localCacheDir, 'ms-playwright', 'daemon', 'daemon', socketDirHash);
 })();
 
+function spawnDaemon(socketPath: string, userDataDir: string, options: SpawnOptions) {
+  const cliPath = path.join(__dirname, '../../../cli.js');
+  debugCli(`Will launch daemon process: ${cliPath}`);
+  return spawn(process.execPath, [cliPath, 'run-mcp-server', `--daemon=${socketPath}`, `--user-data-dir=${userDataDir}`], options);
+}
+
 async function main() {
   const argv = process.argv.slice(2);
   const args = require('minimist')(argv);
diff --git a/packages/playwright/src/mcp/terminal/command.ts b/packages/playwright/src/mcp/terminal/command.ts
@@ -25,7 +25,7 @@ export type CommandSchema<Args extends zodType.ZodTypeAny, Options extends zodTy
   description: string;
   args?: Args;
   options?: Options;
-  toolName: string | ((args: zodType.infer<Args>, options: zodType.infer<Options>) => string);
+  toolName: string | ((args: zodType.infer<Args> & zodType.infer<Options>) => string);
   toolParams: (args: zodType.infer<Args> & zodType.infer<Options>) => any;
 };
 
@@ -51,7 +51,7 @@ export function parseCommand(command: AnyCommandSchema, args: Record<string, str
     throw new Error(formatZodError(e as zodType.ZodError));
   }
 
-  const toolName = typeof command.toolName === 'function' ? command.toolName(parsedArgsObject, options) : command.toolName;
+  const toolName = typeof command.toolName === 'function' ? command.toolName({ ...parsedArgsObject, ...options }) : command.toolName;
   const toolParams = command.toolParams({ ...parsedArgsObject, ...options });
   return { toolName, toolParams };
 }
diff --git a/packages/playwright/src/mcp/terminal/commands.ts b/packages/playwright/src/mcp/terminal/commands.ts
@@ -227,7 +227,7 @@ const fill = declareCommand({
   options: z.object({
     submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
   }),
-  toolName: 'browser_fill',
+  toolName: 'browser_type',
   toolParams: ({ ref, text, submit }) => ({ ref, text, submit }),
 });
 
@@ -242,16 +242,16 @@ const hover = declareCommand({
   toolParams: ({ ref }) => ({ ref }),
 });
 
-const selectOption = declareCommand({
+const select = declareCommand({
   name: 'select',
   description: 'Select an option in a dropdown',
   category: 'core',
   args: z.object({
     ref: z.string().describe('Exact target element reference from the page snapshot'),
-    vals: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),
+    val: z.string().describe('Value to select in the dropdown'),
   }),
   toolName: 'browser_select_option',
-  toolParams: ({ ref, vals: values }) => ({ ref, values }),
+  toolParams: ({ ref, val: value }) => ({ ref, values: [value] }),
 });
 
 const fileUpload = declareCommand({
@@ -308,7 +308,7 @@ const evaluate = declareCommand({
     ref: z.string().optional().describe('Exact target element reference from the page snapshot'),
   }),
   toolName: 'browser_evaluate',
-  toolParams: ({ func: fn, ref }) => ({ function: fn, ref }),
+  toolParams: ({ func, ref }) => ({ function: func, ref }),
 });
 
 const dialogAccept = declareCommand({
@@ -398,10 +398,10 @@ const screenshot = declareCommand({
   }),
   options: z.object({
     filename: z.string().optional().describe('File name to save the screenshot to. Defaults to `page-{timestamp}.{png|jpeg}` if not specified.'),
-    fullPage: z.boolean().optional().describe('When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.'),
+    ['full-page']: z.boolean().optional().describe('When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.'),
   }),
   toolName: 'browser_take_screenshot',
-  toolParams: ({ ref, filename, fullPage }) => ({ filename, ref, fullPage }),
+  toolParams: ({ ref, filename, ['full-page']: fullPage }) => ({ filename, ref, fullPage }),
 });
 
 const pdfSave = declareCommand({
@@ -428,8 +428,8 @@ const consoleList = declareCommand({
   options: z.object({
     clear: z.boolean().optional().describe('Whether to clear the console list'),
   }),
-  toolName: 'browser_console_messages',
-  toolParams: ({ ['min-level']: minLevel }) => ({ minLevel }),
+  toolName: ({ clear }) => clear ? 'browser_console_clear' : 'browser_console_messages',
+  toolParams: ({ ['min-level']: level, clear }) => clear ? ({}) : ({ level }),
 });
 
 const networkRequests = declareCommand({
@@ -441,8 +441,8 @@ const networkRequests = declareCommand({
     static: z.boolean().optional().describe('Whether to include successful static resources like images, fonts, scripts, etc. Defaults to false.'),
     clear: z.boolean().optional().describe('Whether to clear the network list'),
   }),
-  toolName: 'browser_network_requests',
-  toolParams: ({ static: includeStatic }) => ({ includeStatic }),
+  toolName: ({ clear }) => clear ? 'browser_network_clear' : 'browser_network_requests',
+  toolParams: ({ static: includeStatic, clear }) => clear ? ({}) : ({ includeStatic }),
 });
 
 const runCode = declareCommand({
@@ -525,7 +525,7 @@ const commandsArray: AnyCommandSchema[] = [
   fill,
   drag,
   hover,
-  selectOption,
+  select,
   fileUpload,
   check,
   uncheck,
diff --git a/packages/playwright/src/mcp/terminal/help.json b/packages/playwright/src/mcp/terminal/help.json
diff --git a/tests/mcp/cli-fixtures.ts b/tests/mcp/cli-fixtures.ts
diff --git a/tests/mcp/cli.spec.ts b/tests/mcp/cli.spec.ts
diff --git a/tests/mcp/dialogs.spec.ts b/tests/mcp/dialogs.spec.ts
diff --git a/tests/mcp/snapshot-mode.spec.ts b/tests/mcp/snapshot-mode.spec.ts
diff --git a/tests/mcp/terminal.spec.ts b/tests/mcp/terminal.spec.ts

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ const requests = defineTabTool({`
`42`	`42`	`if (rendered)`
`43`	`43`	`text.push(rendered);`
`44`	`44`	`}`
`45`		`- await response.addResult({ text: text.join('\n'), suggestedFilename: params.filename });`
	`45`	`+ await response.addResult({ text: text.join('\n'), suggestedFilename: params.filename, title: 'Network' });`
`46`	`46`	`},`
`47`	`47`	`});`
`48`	`48`