Add image support to readFile via toModelOutput

AGENTS.npm.md

@@ -111,7 +111,7 @@ const { bash } = await createBashTool({
 
 - **just-bash is simulated** - Cannot support python, node.js or binaries; use @vercel/sandbox for a full VM. So, createBashTool supports full VMs, it is just the default that does not
 - **No persistent state between calls** - Each `createBashTool` starts fresh, but the tool itself has persistence and it can be achieved across serverless function invocations by passing in the same sandbox across `createBashTool` invocations
-- **Text files only** - `files` option accepts strings, not buffers
+- **Text files only for `files` option** - `files` accepts strings, not buffers. However, `readFile` returns image files (png, jpg, jpeg, gif, webp) as visual content the model can see when `readFileBuffer` is available on the sandbox.
 - **No streaming** - Command output returned after completion
 
 ## Error Handling

README.md

@@ -55,15 +55,19 @@ Execute bash commands in the sandbox environment. For analysis agents, this may
 
 ### `readFile`
 
-Read the contents of a file from the sandbox.
+Read the contents of a file from the sandbox. For image files (png, jpg,
+jpeg, gif, webp), returns visual content the model can see when the sandbox
+supports `readFileBuffer`.
 
 **Input:**
 
 - `path` (string): The path to the file to read
 
 **Returns:**
 
-- `content` (string): The file contents
+- `content` (string): The file contents for text files
+- `data` (string): Base64-encoded image data for supported image files
+- `mediaType` (string): The image MIME type
 
 ### `writeFile`
 
@@ -170,6 +174,10 @@ const customSandbox: Sandbox = {
   async writeFiles(files) {
     // Your implementation here - files is Array<{path, content}>
   },
+  async readFileBuffer(path) {
+    // Optional: implement to enable image support in readFile
+    return new Uint8Array();
+  },
 };
 
 const { tools } = await createBashTool({ sandbox: customSandbox });

src/sandbox/just-bash.ts

@@ -12,6 +12,7 @@ export interface JustBashLike {
   }>;
   fs: {
     readFile: (path: string) => Promise<string>;
+    readFileBuffer?: (path: string) => Promise<Uint8Array>;
     writeFile: (path: string, content: string) => Promise<void>;
   };
 }
@@ -91,6 +92,13 @@ export async function createJustBashSandbox(
       return bashEnv.fs.readFile(filePath);
     },
 
+    async readFileBuffer(filePath: string): Promise<Uint8Array> {
+      return (
+        (await bashEnv.fs.readFileBuffer?.(filePath)) ??
+        new TextEncoder().encode(await bashEnv.fs.readFile(filePath))
+      );
+    },
+
     async writeFiles(
       files: Array<{ path: string; content: string | Buffer }>,
     ): Promise<void> {
@@ -133,6 +141,13 @@ export function wrapJustBash(bashInstance: JustBashLike): Sandbox {
       return bashInstance.fs.readFile(filePath);
     },
 
+    async readFileBuffer(filePath: string): Promise<Uint8Array> {
+      return (
+        (await bashInstance.fs.readFileBuffer?.(filePath)) ??
+        new TextEncoder().encode(await bashInstance.fs.readFile(filePath))
+      );
+    },
+
     async writeFiles(
       files: Array<{ path: string; content: string | Buffer }>,
     ): Promise<void> {

src/sandbox/vercel.ts

@@ -70,6 +70,18 @@ export function wrapVercelSandbox(vercelSandbox: VercelSandboxLike): Sandbox {
       return streamToString(stream);
     },
 
+    async readFileBuffer(filePath: string): Promise<Uint8Array> {
+      const stream = await vercelSandbox.readFile({ path: filePath });
+      if (stream === null) {
+        throw new Error(`File not found: ${filePath}`);
+      }
+      const chunks: Buffer[] = [];
+      for await (const chunk of stream) {
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+      }
+      return Buffer.concat(chunks);
+    },
+
     async writeFiles(
       files: Array<{ path: string; content: string | Buffer }>,
     ): Promise<void> {

src/tool.test.ts

@@ -11,6 +11,7 @@ vi.mock("ai", () => ({
     description: config.description,
     inputSchema: config.inputSchema,
     execute: config.execute,
+    toModelOutput: config.toModelOutput,
   })),
 }));
 
@@ -26,6 +27,7 @@ vi.mock("just-bash", async () => {
     Bash: class MockBash {
       fs: {
         readFile: (path: string) => Promise<string>;
+        readFileBuffer: (path: string) => Promise<Uint8Array>;
         writeFile: (path: string, content: string) => Promise<void>;
       };
 
@@ -35,9 +37,12 @@ vi.mock("just-bash", async () => {
 
         this.fs = {
           readFile: async (path: string) => {
-            if (mockFiles[path]) {
-              return mockFiles[path];
-            }
+            if (mockFiles[path]) return mockFiles[path];
+            throw new Error(`ENOENT: no such file: ${path}`);
+          },
+          readFileBuffer: async (path: string) => {
+            if (mockFiles[path])
+              return new TextEncoder().encode(mockFiles[path]);
             throw new Error(`ENOENT: no such file: ${path}`);
           },
           writeFile: async (path: string, content: string) => {
@@ -334,6 +339,67 @@ describe("createBashTool", () => {
     expect(result.stdout).toBe("custom");
   });
 
+  it("readFile returns image data for png files", async () => {
+    const pngBytes = new Uint8Array([0x01, 0x02, 0x03]);
+    mockFiles["/workspace/chart.png"] = String.fromCharCode(...pngBytes);
+
+    const { tools } = await createBashTool();
+    assert(tools.readFile.execute, "readFile.execute should be defined");
+    const result = (await tools.readFile.execute(
+      { path: "/workspace/chart.png" },
+      opts,
+    )) as { data: string; mediaType: string };
+    expect(result.mediaType).toBe("image/png");
+    expect(result.data).toBe(Buffer.from(pngBytes).toString("base64"));
+  });
+
+  it("readFile toModelOutput returns image-data for images", async () => {
+    const { tools } = await createBashTool();
+    assert(tools.readFile.toModelOutput, "toModelOutput should be defined");
+    const result = tools.readFile.toModelOutput({
+      toolCallId: "test",
+      input: { path: "x.png" },
+      output: { data: "abc123", mediaType: "image/png" },
+    });
+    expect(result).toEqual({
+      type: "content",
+      value: [{ type: "image-data", data: "abc123", mediaType: "image/png" }],
+    });
+  });
+
+  it("readFile toModelOutput returns json for text files", async () => {
+    const { tools } = await createBashTool();
+    assert(tools.readFile.toModelOutput, "toModelOutput should be defined");
+    const result = tools.readFile.toModelOutput({
+      toolCallId: "test",
+      input: { path: "x.txt" },
+      output: { content: "hello" },
+    });
+    expect(result).toEqual({
+      type: "json",
+      value: { content: "hello" },
+    });
+  });
+
+  it("readFile falls back to text when sandbox lacks readFileBuffer", async () => {
+    // The mock will have readFileBuffer via our update, so test the fallback
+    // by creating a custom sandbox without it
+    const { createBashTool: realCreate } = await import("./tool.js");
+    const customSandbox = {
+      executeCommand: async () => ({ stdout: "", stderr: "", exitCode: 0 }),
+      readFile: async () => "text-content",
+      writeFiles: async () => {},
+      // no readFileBuffer
+    };
+    const { tools } = await realCreate({ sandbox: customSandbox });
+    assert(tools.readFile.execute, "readFile.execute should be defined");
+    const result = (await tools.readFile.execute(
+      { path: "/workspace/test.png" },
+      opts,
+    )) as { content: string };
+    expect(result.content).toBe("text-content");
+  });
+
   it("writes files in batches of 20 to custom sandbox", async () => {
     const customSandbox = {
       executeCommand: vi

src/tools/read-file.ts

@@ -3,6 +3,14 @@ import { tool } from "ai";
 import { z } from "zod";
 import type { Sandbox } from "../types.js";
 
+const IMAGE_TYPES: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+};
+
 const readFileSchema = z.object({
   path: z.string().describe("The path to the file to read"),
 });
@@ -17,12 +25,45 @@ export function createReadFileTool(options: CreateReadFileToolOptions) {
   const { sandbox, cwd } = options;
 
   return tool({
-    description: "Read the contents of a file from the sandbox.",
+    description:
+      "Read the contents of a file from the sandbox. " +
+      "For image files (png, jpg, jpeg, gif, webp), returns visual content the model can see.",
     inputSchema: readFileSchema,
     execute: async ({ path }) => {
       const resolvedPath = nodePath.posix.resolve(cwd, path);
+      const ext = nodePath.extname(resolvedPath).toLowerCase();
+      const mediaType = IMAGE_TYPES[ext];
+
+      if (mediaType && sandbox.readFileBuffer) {
+        const buffer = await sandbox.readFileBuffer(resolvedPath);
+        const base64 = Buffer.from(buffer).toString("base64");
+        return { data: base64, mediaType };
+      }
+
       const content = await sandbox.readFile(resolvedPath);
       return { content };
     },
+
+    toModelOutput({
+      output,
+    }: {
+      toolCallId: string;
+      input: unknown;
+      output: Record<string, unknown>;
+    }) {
+      if ("data" in output) {
+        return {
+          type: "content" as const,
+          value: [
+            {
+              type: "image-data" as const,
+              data: output.data as string,
+              mediaType: output.mediaType as string,
+            },
+          ],
+        };
+      }
+      return { type: "json" as const, value: output as never };
+    },
   });
 }

src/types.ts

@@ -11,6 +11,7 @@ export interface CommandResult {
 export interface Sandbox {
   executeCommand(command: string): Promise<CommandResult>;
   readFile(path: string): Promise<string>;
+  readFileBuffer?(path: string): Promise<Uint8Array>;
   writeFiles(
     files: Array<{ path: string; content: string | Buffer }>,
   ): Promise<void>;