diff --git a/report-app/src/app/pages/report-viewer/report-viewer.html b/report-app/src/app/pages/report-viewer/report-viewer.html index 548ca1f..b1693e4 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.html +++ b/report-app/src/app/pages/report-viewer/report-viewer.html @@ -218,7 +218,7 @@

Servers

diff --git a/runner/codegen/genkit/genkit-runner.ts b/runner/codegen/genkit/genkit-runner.ts index d5e36e6..1e5983d 100644 --- a/runner/codegen/genkit/genkit-runner.ts +++ b/runner/codegen/genkit/genkit-runner.ts @@ -1,4 +1,11 @@ -import {DynamicResourceAction, GenerateResponse, genkit, ModelReference, ToolAction} from 'genkit'; +import { + Action, + DynamicResourceAction, + GenerateResponse, + genkit, + ModelReference, + ToolAction, +} from 'genkit'; import {GenkitMcpHost, McpServerConfig, createMcpHost} from '@genkit-ai/mcp'; import {GenkitPlugin, GenkitPluginV2} from 'genkit/plugin'; import {z} from 'zod'; @@ -11,6 +18,7 @@ import { LocalLlmGenerateTextResponse, LocalLlmGenerateTextRequestOptions, LocalLlmGenerateFilesRequestOptions, + McpServerDetails, } from '../llm-runner.js'; import {setTimeout} from 'node:timers/promises'; import {callWithTimeout} from '../../utils/timeout.js'; @@ -21,10 +29,18 @@ import {UserFacingError} from '../../utils/errors.js'; import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js'; import {ToolLogEntry} from '../../shared-interfaces.js'; import {combineAbortSignals} from '../../utils/abort-signal.js'; +import {toToolDefinition} from 'genkit/tool'; const globalLogger = new GenkitLogger(); logger.init(globalLogger); +/** + * Gets the name of a Genkit action. + */ +function getActionName(action: Action): string { + return toToolDefinition(action).name; +} + /** Runner that uses the Genkit API under the hood. */ export class GenkitRunner implements LlmRunner { readonly id = 'genkit'; @@ -199,7 +215,10 @@ export class GenkitRunner implements LlmRunner { } } - startMcpServerHost(hostName: string, servers: McpServerOptions[]): void { + async startMcpServerHost( + hostName: string, + servers: McpServerOptions[], + ): Promise { if (this.mcpHost !== null) { throw new Error('MCP host is already started'); } @@ -216,6 +235,12 @@ export class GenkitRunner implements LlmRunner { globalLogger.startCapturingLogs(); this.mcpHost = createMcpHost({name: hostName, mcpServers}); + const tools = await this.mcpHost.getActiveTools(this.genkitInstance); + const resources = await this.mcpHost.getActiveResources(this.genkitInstance); + return { + tools: tools.map(getActionName), + resources: resources.map(getActionName), + }; } flushMcpServerLogs(): string[] { diff --git a/runner/codegen/llm-runner.ts b/runner/codegen/llm-runner.ts index 6946e8c..75baca0 100644 --- a/runner/codegen/llm-runner.ts +++ b/runner/codegen/llm-runner.ts @@ -58,8 +58,9 @@ export interface LlmRunner { * Optional since not all runners may support MCP. * @param hostName Name for the MCP host. * @param servers Configured servers that should be started. + * @returns Details about the created server. */ - startMcpServerHost?(hostName: string, servers: McpServerOptions[]): void; + startMcpServerHost?(hostName: string, servers: McpServerOptions[]): Promise; /** Stops tracking MCP server logs and returns the current ones. */ flushMcpServerLogs?(): string[]; @@ -171,6 +172,12 @@ export const mcpServerOptionsSchema = z.object({ /** Options used to start an MCP server. */ export type McpServerOptions = z.infer; +/** Details about an MCP server. */ +export interface McpServerDetails { + tools: string[]; + resources: string[]; +} + /** * Type for a prompt message may be passed to LLM runner in the eval tool. * diff --git a/runner/orchestration/executors/local-executor.ts b/runner/orchestration/executors/local-executor.ts index 80696eb..a810a9c 100644 --- a/runner/orchestration/executors/local-executor.ts +++ b/runner/orchestration/executors/local-executor.ts @@ -1,7 +1,7 @@ import {ChildProcess, fork} from 'node:child_process'; import path, {join} from 'node:path'; import PQueue from 'p-queue'; -import {LlmRunner} from '../../codegen/llm-runner.js'; +import {LlmRunner, McpServerDetails} from '../../codegen/llm-runner.js'; import {getRunnerByName, RunnerName} from '../../codegen/runner-creation.js'; import {ProgressLogger} from '../../progress/progress-logger.js'; import { @@ -225,16 +225,16 @@ export class LocalExecutor implements Executor { }; } - async startMcpServerHost(hostName: string) { + async startMcpServerHost(hostName: string): Promise { const llm = await this.llm; if (llm.startMcpServerHost === undefined) { - return; + return undefined; } - llm.startMcpServerHost(hostName, this.config.mcpServers ?? []); + return llm.startMcpServerHost(hostName, this.config.mcpServers ?? []); } - async collectMcpServerLogs() { + async collectMcpServerLogs(mcpServerDetails: McpServerDetails | undefined) { const llm = await this.llm; if (llm.flushMcpServerLogs === undefined) { return; @@ -245,6 +245,8 @@ export class LocalExecutor implements Executor { name: m.name, command: m.command, args: m.args, + tools: mcpServerDetails?.tools ?? [], + resources: mcpServerDetails?.resources ?? [], })), logs: llm.flushMcpServerLogs().join('\n'), }; diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts index bcbd962..4da72c3 100644 --- a/runner/orchestration/generate.ts +++ b/runner/orchestration/generate.ts @@ -85,9 +85,10 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise< // We need Chrome to collect runtime information. await installChrome(); - if (options.startMcp && env.executor instanceof LocalExecutor) { - env.executor.startMcpServerHost(`mcp-${env.clientSideFramework.id}`); - } + const mcpServerDetails = + env.executor instanceof LocalExecutor && options.startMcp && env.executor.startMcpServerHost + ? await env.executor.startMcpServerHost(`mcp-${env.clientSideFramework.id}`) + : undefined; progress.initialize(promptsToProcess.length); @@ -163,7 +164,7 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise< const mcp = env.executor instanceof LocalExecutor && options.startMcp - ? await env.executor.collectMcpServerLogs() + ? await env.executor.collectMcpServerLogs(mcpServerDetails) : undefined; const timestamp = new Date(); diff --git a/runner/shared-interfaces.ts b/runner/shared-interfaces.ts index 0b19326..68c3dc1 100644 --- a/runner/shared-interfaces.ts +++ b/runner/shared-interfaces.ts @@ -437,7 +437,15 @@ export interface RunDetails { /** Information about configured MCP servers, if any. */ mcp?: { /** MCP servers that were configured. */ - servers: {name: string; command: string; args: string[]}[]; + servers: { + name: string; + command: string; + args: string[]; + /** Tools reported for this server. */ + tools?: string[]; + /** Resources reported for this server. */ + resources?: string[]; + }[]; /** Logs produced by all of the servers. */ logs: string;