From 043dc7afaf9c3c66c5abd2b14b9de641fde03368 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Tue, 28 Jul 2026 15:16:02 +0300 Subject: [PATCH] fix(server): return capture_ui_snapshot images as ImageContent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit capture_ui_snapshot registered no onSuccess transform, so the whole bundle — including the base64 frames under screenshots.images — was serialised into a single TextContent. One 1512x882 desktop frame is ~145 KB of base64, which overflows the response budget of MCP clients that cap tool output, making the documented "unified evidence" entry point unusable over MCP (the CLI path is unaffected: it prints the CoreResult envelope and never reaches this layer). Lift the images out of the bundle into ImageContent artifacts, the same contract get_screenshots already uses, and emit the remaining bundle JSON as text with screenshots.images emptied. imageSummaries still carries the per-image ids and hashes, so the JSON stays self-describing. The fileUrls branch is untouched: with --save-images there are no inline images and the bundle remains a single TextContent. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/src/tools/inspection_tools.dart | 37 ++++++++-- .../test/tools/inspection_tools_test.dart | 67 +++++++++++++++++++ 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/packages/server_capability_core/lib/src/tools/inspection_tools.dart b/packages/server_capability_core/lib/src/tools/inspection_tools.dart index fca3cc49..9d8c08b9 100644 --- a/packages/server_capability_core/lib/src/tools/inspection_tools.dart +++ b/packages/server_capability_core/lib/src/tools/inspection_tools.dart @@ -156,10 +156,12 @@ void registerInspectionTools(final CapabilityContext context) { // --------------------------------------------------------------------------- // capture_ui_snapshot // --------------------------------------------------------------------------- - // The "bundle" described in docs is JSON inside a single TextContent. - // Legacy resource_handler.dart captureUiSnapshot (lines 472-474) returns - // CallToolResult(content: [TextContent(text: jsonEncode(result.data))]). - // No multi-content transform required — standard runCommand with no onSuccess. + // The bundle travels as JSON in a TextContent, but captured images are lifted + // out of `screenshots.images` into ImageContent blocks (same artifact contract + // as get_screenshots). Base64 inlined into the JSON counts against the client + // response budget as text, which overflows it for a single desktop frame. + // `imageSummaries` keeps the per-image ids/hashes, so the bundle stays + // self-describing with the payload removed. context.registerTool( ToolRegistration( name: 'capture_ui_snapshot', @@ -188,6 +190,33 @@ void registerInspectionTools(final CapabilityContext context) { screenshotMode: parseScreenshotMode(args['screenshotMode']), permissionPolicy: parsePermissionPolicy(args['permissionPolicy']), ), + onSuccess: (final data) { + final bundle = _asMap(data); + final screenshots = _asMap(bundle['screenshots']); + final images = _stringList(screenshots['images']); + if (images.isEmpty) { + return AgentResult.success( + artifacts: [AgentArtifact.text(jsonEncode(bundle))], + ); + } + return AgentResult.success( + artifacts: [ + AgentArtifact.text( + jsonEncode({ + ...bundle, + 'screenshots': { + ...screenshots, + 'images': const [], + }, + }), + ), + ...images.map( + (final image) => + AgentArtifact.text(image, mimeType: 'image/png'), + ), + ], + ); + }, ); }, ), diff --git a/packages/server_capability_core/test/tools/inspection_tools_test.dart b/packages/server_capability_core/test/tools/inspection_tools_test.dart index 064285e4..332b06cd 100644 --- a/packages/server_capability_core/test/tools/inspection_tools_test.dart +++ b/packages/server_capability_core/test/tools/inspection_tools_test.dart @@ -591,6 +591,73 @@ void main() { }, ); + test( + 'capture_ui_snapshot onSuccess: images move to ImageContent blocks ' + 'and leave the bundle JSON', + () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success( + data: { + 'message': 'Captured UI snapshot bundle.', + 'screenshots': { + 'images': ['base64dataA', 'base64dataB'], + 'fileUrls': [], + 'captureMode': 'flutter_layer', + }, + 'imageSummaries': [ + {'id': 'image_1', 'source': 'inline_base64'}, + {'id': 'image_2', 'source': 'inline_base64'}, + ], + }, + ); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('capture_ui_snapshot')!; + final result = await reg.handler(const {}); + expect(result.ok, isTrue); + expect(result.artifacts, hasLength(3)); + + final bundle = + jsonDecode(result.artifacts.first.text!) as Map; + final screenshots = bundle['screenshots']! as Map; + expect(screenshots['images'], isEmpty); + expect(screenshots['captureMode'], 'flutter_layer'); + expect(bundle['imageSummaries'], hasLength(2)); + + final images = result.artifacts + .where((final a) => a.mimeType == 'image/png') + .toList(); + expect(images, hasLength(2)); + expect(images[0].text, 'base64dataA'); + expect(images[1].text, 'base64dataB'); + }, + ); + + test( + 'capture_ui_snapshot onSuccess: fileUrls bundle stays a single ' + 'TextContent', + () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success( + data: { + 'screenshots': { + 'images': [], + 'fileUrls': ['file:///tmp/screen0.png'], + }, + }, + ); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('capture_ui_snapshot')!; + final result = await reg.handler(const {}); + expect(result.ok, isTrue); + expect(result.artifacts, hasLength(1)); + expect(result.artifacts.single.mimeType, 'text/plain'); + final bundle = + jsonDecode(result.artifacts.single.text!) as Map; + final screenshots = bundle['screenshots']! as Map; + expect(screenshots['fileUrls'], ['file:///tmp/screen0.png']); + }, + ); + test( 'capture_ui_snapshot handler short-circuits on override failure', () async {