diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4fee0e0d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - name: Unit tests + run: npm test --workspace @onkernel/cua-ai -- test/models.test.ts test/tools.test.ts + + integration: + runs-on: ubuntu-latest + timeout-minutes: 15 + # Only run on the main repo (not forks) so secrets are available. + if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - name: Integration tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} + YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} + run: npm test --workspace @onkernel/cua-ai -- test/batch-tool.integration.test.ts diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index ee42659c..c882c808 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -245,6 +245,7 @@ const CUA_ACTION_SCHEMA_BY_TYPE = { export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): TSchema { if (actions.length === 0) throw new Error("actions must include at least one CUA action type"); + if (actions.length === 1) return CUA_ACTION_SCHEMA_BY_TYPE[actions[0]!]; return Type.Union(actions.map((action) => CUA_ACTION_SCHEMA_BY_TYPE[action])); } diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index 45a79cd9..0785f229 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -47,6 +47,8 @@ export const streamTzafonResponses: StreamFunction { - if (typeof value === "string" && value.trim()) { - try { - const parsed = JSON.parse(value); - return parsed && typeof parsed === "object" ? parsed as Record : {}; - } catch { - return {}; - } + const top = + typeof value === "string" && value.trim() + ? safeJsonParse(value) + : value && typeof value === "object" + ? (value as Record) + : {}; + if (!top || typeof top !== "object") return {}; + // Tzafon sometimes nests JSON-encoded arrays/objects inside the top-level argument object + // (observed: { "actions": "[{...}]" }). Unwrap one level so consumers get real values. + const out: Record = {}; + for (const [key, val] of Object.entries(top)) { + out[key] = typeof val === "string" && looksLikeJson(val) ? safeJsonParse(val) ?? val : val; } - if (value && typeof value === "object") return value as Record; - return {}; + return out; +} + +function safeJsonParse(value: string): Record | unknown[] | null { + try { + const parsed = JSON.parse(value); + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function looksLikeJson(value: string): boolean { + const trimmed = value.trim(); + return trimmed.startsWith("[") || trimmed.startsWith("{"); +} + +function usageFromTzafon(usage: unknown): AssistantMessage["usage"] { + const input = readNumber(usage, "input_tokens"); + const output = readNumber(usage, "output_tokens"); + const cacheRead = readNumber(getValue(usage, "input_tokens_details"), "cached_tokens"); + const totalTokens = readNumber(usage, "total_tokens") || input + output; + return { + input, + output, + cacheRead, + cacheWrite: 0, + totalTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function readNumber(obj: unknown, key: string): number { + if (!obj || typeof obj !== "object") return 0; + const n = (obj as Record)[key]; + return typeof n === "number" && Number.isFinite(n) ? n : 0; } function getArray(obj: unknown, key: string): unknown[] { diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index e90027a5..2588d56e 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -13,7 +13,7 @@ import { type TextContent, type ToolCall, } from "@earendil-works/pi-ai"; -import { CUA_ACTION_TYPES } from "../common.js"; +import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, type CuaAction } from "../common.js"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; @@ -92,14 +92,25 @@ async function runYutoriStream( const text = typeof message?.content === "string" ? message.content : ""; if (text) emitText(stream, output, text); + const wantsBatch = (context.tools ?? []).some((tool) => tool.name === CUA_BATCH_TOOL_NAME); + const batchActions: CuaAction[] = []; + let firstBatchCallId: string | undefined; + for (const call of message?.tool_calls ?? []) { if (call.type !== "function") continue; + const args = parseArguments(call.function.arguments); + const canonical = wantsBatch ? toCanonicalAction(call.function.name, args) : undefined; + if (canonical) { + batchActions.push(...canonical); + firstBatchCallId ??= call.id; + continue; + } const contentIndex = output.content.length; const toolCall: ToolCall = { type: "toolCall", id: call.id, name: call.function.name, - arguments: parseArguments(call.function.arguments), + arguments: args, }; output.content.push(toolCall); stream.push({ type: "toolcall_start", contentIndex, partial: output }); @@ -107,6 +118,21 @@ async function runYutoriStream( stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); } + if (batchActions.length > 0) { + const contentIndex = output.content.length; + const toolCall: ToolCall = { + type: "toolCall", + id: firstBatchCallId ?? `yutori_batch_${Date.now()}`, + name: CUA_BATCH_TOOL_NAME, + arguments: { actions: batchActions }, + }; + output.content.push(toolCall); + output.stopReason = "toolUse"; + stream.push({ type: "toolcall_start", contentIndex, partial: output }); + stream.push({ type: "toolcall_delta", contentIndex, delta: JSON.stringify(toolCall.arguments), partial: output }); + stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output }); + } + stream.push({ type: "done", reason: output.stopReason as "stop" | "length" | "toolUse", message: output }); stream.end(); } catch (err) { @@ -206,6 +232,72 @@ function parseArguments(value: string | undefined): Record { return JSON.parse(value) as Record; } +const SCROLL_AMOUNT_PER_NOTCH = 120; + +function readPoint(value: unknown): { x: number; y: number } | undefined { + if (!Array.isArray(value) || value.length < 2) return undefined; + const x = Number(value[0]); + const y = Number(value[1]); + if (!Number.isFinite(x) || !Number.isFinite(y)) return undefined; + return { x, y }; +} + +function toCanonicalAction(name: string, args: Record): CuaAction[] | undefined { + const coords = readPoint(args.coordinates); + switch (name) { + case "left_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y }] : undefined; + case "right_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "right" }] : undefined; + case "middle_click": + return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "middle" }] : undefined; + case "double_click": + return coords ? [{ type: "double_click", x: coords.x, y: coords.y }] : undefined; + case "mouse_move": + case "hover": + return coords ? [{ type: "move", x: coords.x, y: coords.y }] : undefined; + case "mouse_down": + return coords ? [{ type: "mouse_down", x: coords.x, y: coords.y }] : undefined; + case "mouse_up": + return coords ? [{ type: "mouse_up", x: coords.x, y: coords.y }] : undefined; + case "type": { + const text = typeof args.text === "string" ? args.text : undefined; + return text !== undefined ? [{ type: "type", text }] : undefined; + } + case "key_press": + case "hold_key": { + const key = typeof args.key === "string" ? args.key : undefined; + return key ? [{ type: "keypress", keys: [key] }] : undefined; + } + case "scroll": { + if (!coords) return undefined; + const amount = typeof args.amount === "number" ? args.amount : 1; + const direction = typeof args.direction === "string" ? args.direction : "down"; + const ticks = amount * SCROLL_AMOUNT_PER_NOTCH; + const dx = direction === "left" ? -ticks : direction === "right" ? ticks : 0; + const dy = direction === "up" ? -ticks : direction === "down" ? ticks : 0; + return [{ type: "scroll", x: coords.x, y: coords.y, scroll_x: dx, scroll_y: dy }]; + } + case "drag": { + const start = readPoint(args.start_coordinates); + if (!start || !coords) return undefined; + return [{ type: "drag", path: [start, coords] }]; + } + case "wait": + return [{ type: "wait" }]; + case "go_back": + return [{ type: "back" }]; + case "go_forward": + return [{ type: "forward" }]; + case "goto_url": { + const url = typeof args.url === "string" ? args.url : undefined; + return url ? [{ type: "goto", url }] : undefined; + } + default: + return undefined; + } +} + function usageFromYutori(usage: unknown): AssistantMessage["usage"] { const input = readNumber(usage, "prompt_tokens"); const output = readNumber(usage, "completion_tokens"); diff --git a/packages/ai/test/batch-tool.integration.test.ts b/packages/ai/test/batch-tool.integration.test.ts new file mode 100644 index 00000000..f5af9aba --- /dev/null +++ b/packages/ai/test/batch-tool.integration.test.ts @@ -0,0 +1,246 @@ +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + CUA_ACTION_TYPES, + CUA_BATCH_TOOL_NAME, + type Context, + type CuaActionType, + type CuaProvider, + anthropic, + complete, + gemini, + getCuaModel, + openai, + tzafon, + yutori, +} from "../src/index.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const screenshotPath = join(here, "..", "examples", "screenshot.png"); + +interface ProviderCase { + provider: CuaProvider; + envVar: string; + modelRef: string; + tools: () => ReturnType; + multiActionTools: () => ReturnType; + coordinateRange: readonly [number, number]; + supportsBatching: boolean; + extraOptions?: Record; +} + +const cases: ProviderCase[] = [ + { + provider: "openai", + envVar: "OPENAI_API_KEY", + modelRef: "openai:gpt-5.5", + tools: () => openai.createComputerToolDefinitions({ actions: ["click"] }), + multiActionTools: () => openai.createComputerToolDefinitions({ actions: ["click", "type"] }), + coordinateRange: [0, 1920], + supportsBatching: true, + }, + { + provider: "anthropic", + envVar: "ANTHROPIC_API_KEY", + modelRef: "anthropic:claude-opus-4-7", + tools: () => anthropic.createComputerToolDefinitions({ actions: ["click"] }), + multiActionTools: () => anthropic.createComputerToolDefinitions({ actions: ["click", "type"] }), + coordinateRange: [0, 1920], + supportsBatching: true, + }, + { + provider: "gemini", + envVar: "GOOGLE_API_KEY", + modelRef: "gemini:gemini-3-flash-preview", + tools: () => gemini.createComputerToolDefinitions({ actions: ["click"] }), + multiActionTools: () => gemini.createComputerToolDefinitions({ actions: ["click", "type"] }), + coordinateRange: [0, 999], + supportsBatching: true, + }, + { + provider: "tzafon", + envVar: "TZAFON_API_KEY", + modelRef: "tzafon:tzafon.northstar-cua-fast", + tools: () => tzafon.createComputerToolDefinitions({ actions: ["click"] }), + multiActionTools: () => tzafon.createComputerToolDefinitions({ actions: ["click", "type"] }), + coordinateRange: [0, 999], + supportsBatching: true, + }, + { + provider: "yutori", + envVar: "YUTORI_API_KEY", + modelRef: "yutori:n1.5-latest", + tools: () => yutori.createComputerToolDefinitions({ actions: ["click"] }), + multiActionTools: () => yutori.createComputerToolDefinitions({ actions: ["click", "type"] }), + coordinateRange: [0, 1000], + // Yutori's server-side model always emits one native tool call per response, + // so the translated batch always contains exactly one action. + supportsBatching: false, + }, +]; + +async function buildContext(tools: ProviderCase["tools"]): Promise { + const screenshot = await readFile(screenshotPath); + return { + systemPrompt: [ + "You are controlling a browser from a screenshot.", + "Call batch_computer_actions with one action that clicks the sign in / up link.", + ].join("\n"), + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link in this Kernel homepage screenshot." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools: tools(), + }; +} + +async function buildMultiActionContext(tools: ProviderCase["multiActionTools"]): Promise { + const screenshot = await readFile(screenshotPath); + return { + systemPrompt: [ + "You are controlling a browser from a screenshot.", + "Always batch multiple steps into a single batch_computer_actions call by adding all required actions to the actions array.", + "Do NOT split a sequence across separate tool calls.", + ].join("\n"), + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Click the sign in / up link and then type 'alice@example.com'. Emit BOTH actions in one single batch_computer_actions call.", + }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools: tools(), + }; +} + +describe("batch_computer_actions integration", () => { + for (const c of cases) { + const hasKey = !!process.env[c.envVar]; + const test = hasKey ? it : it.skip; + + test(`${c.provider} returns a batch_computer_actions tool call with parsed actions`, async () => { + const model = getCuaModel(c.modelRef as never); + const context = await buildContext(c.tools); + const response = await complete(model, context, { + apiKey: process.env[c.envVar], + maxTokens: 1024, + ...c.extraOptions, + }); + + const toolCalls = response.content.filter((part) => part.type === "toolCall"); + expect(toolCalls.length, `${c.provider} returned no tool calls`).toBeGreaterThan(0); + + const batch = toolCalls.find((call) => call.name === CUA_BATCH_TOOL_NAME); + expect(batch, `${c.provider} did not return ${CUA_BATCH_TOOL_NAME}; got [${toolCalls.map((c) => c.name).join(", ")}]`).toBeDefined(); + + const args = batch!.arguments as { actions?: unknown }; + expect(Array.isArray(args.actions), `${c.provider} .arguments.actions is ${typeof args.actions}, expected array`).toBe(true); + + const actions = args.actions as Array>; + expect(actions.length).toBeGreaterThan(0); + + const click = actions.find((a) => a.type === "click"); + expect(click, `${c.provider} batch had no click action; got: ${JSON.stringify(actions)}`).toBeDefined(); + expect(typeof click!.x).toBe("number"); + expect(typeof click!.y).toBe("number"); + const [min, max] = c.coordinateRange; + expect(click!.x as number, `${c.provider} x out of range`).toBeGreaterThanOrEqual(min); + expect(click!.x as number).toBeLessThanOrEqual(max); + expect(click!.y as number, `${c.provider} y out of range`).toBeGreaterThanOrEqual(min); + expect(click!.y as number).toBeLessThanOrEqual(max); + + expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThan(0); + + for (const action of actions) { + expect(CUA_ACTION_TYPES).toContain(action.type as CuaActionType); + } + }, 60_000); + } + + const yutoriHasKey = !!process.env.YUTORI_API_KEY; + (yutoriHasKey ? it : it.skip)( + "yutori translates native left_click into a batch_computer_actions call", + async () => { + const model = getCuaModel("yutori:n1.5-latest"); + const context = await buildContext(() => yutori.createComputerToolDefinitions()); + const response = await complete(model, context, { + apiKey: process.env.YUTORI_API_KEY, + maxTokens: 1024, + }); + + const batchCalls = response.content.filter((part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME); + expect(batchCalls.length, "yutori did not emit a translated batch call").toBeGreaterThan(0); + const args = batchCalls[0]!.arguments as { actions: Array> }; + expect(Array.isArray(args.actions)).toBe(true); + expect(args.actions.length).toBeGreaterThan(0); + expect(args.actions[0]!.type).toBe("click"); + }, + 60_000, + ); +}); + +// The whole point of batch_computer_actions is to let a single tool call carry +// multiple ordered actions (click-then-type, type-then-Enter, etc.). These +// tests probe each provider with a two-step task and verify whether the model +// actually packs both steps into one batch call. +describe("batch_computer_actions multi-action sequences", () => { + for (const c of cases) { + const hasKey = !!process.env[c.envVar]; + const test = hasKey ? it : it.skip; + + if (c.supportsBatching) { + test(`${c.provider} packs a click+type sequence into a single batch call`, async () => { + const model = getCuaModel(c.modelRef as never); + const context = await buildMultiActionContext(c.multiActionTools); + const response = await complete(model, context, { + apiKey: process.env[c.envVar], + maxTokens: 2048, + ...c.extraOptions, + }); + + const batchCalls = response.content.filter( + (part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME, + ); + expect(batchCalls.length, `${c.provider} produced no batch_computer_actions calls`).toBe(1); + + const args = batchCalls[0]!.arguments as { actions: Array> }; + expect(Array.isArray(args.actions), `${c.provider} actions field is not an array`).toBe(true); + expect( + args.actions.length, + `${c.provider} only emitted ${args.actions.length} action(s) in one batch call: ${JSON.stringify(args.actions)}`, + ).toBeGreaterThanOrEqual(2); + }, 60_000); + } else { + test(`${c.provider} emits exactly one action per response (model does not batch)`, async () => { + const model = getCuaModel(c.modelRef as never); + const context = await buildMultiActionContext(c.multiActionTools); + const response = await complete(model, context, { + apiKey: process.env[c.envVar], + maxTokens: 2048, + ...c.extraOptions, + }); + + const batchCalls = response.content.filter( + (part) => part.type === "toolCall" && part.name === CUA_BATCH_TOOL_NAME, + ); + expect(batchCalls.length).toBe(1); + const args = batchCalls[0]!.arguments as { actions: Array> }; + expect(args.actions.length).toBe(1); + }, 60_000); + } + } +}); diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index adf7450b..03de9a12 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -1,8 +1,23 @@ import { describe, expect, it } from "vitest"; -import { CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, anthropic, gemini, openai, tzafon, yutori } from "../src/index.js"; +import { + CUA_ACTION_TYPES, + CUA_BATCH_TOOL_NAME, + CUA_NAVIGATION_TOOL_NAME, + type CuaActionType, + anthropic, + gemini, + openai, + tzafon, + yutori, +} from "../src/index.js"; const providers = { openai, anthropic, gemini, tzafon, yutori }; +function batchActionVariants(tool: { parameters: any }): any[] { + const items = tool.parameters.properties.actions.items; + return items.anyOf ?? items.oneOf ?? [items]; +} + describe("computer tool definitions", () => { for (const [provider, namespace] of Object.entries(providers)) { it(`returns a default batch tool for ${provider}`, () => { @@ -14,20 +29,63 @@ describe("computer tool definitions", () => { const tools = namespace.createComputerToolDefinitions({ actions: ["click"] }); expect(tools.map((tool) => tool.name)).toEqual([CUA_BATCH_TOOL_NAME]); }); + + it(`default batch tool for ${provider} covers every CUA action type`, () => { + const tools = namespace.createComputerToolDefinitions(); + const variants = batchActionVariants(tools[0]!); + const seen = variants.map((variant) => variant.properties.type.const).sort(); + expect(seen).toEqual([...CUA_ACTION_TYPES].sort()); + }); + + it(`each action variant for ${provider} accepts only declared fields`, () => { + const tools = namespace.createComputerToolDefinitions(); + for (const variant of batchActionVariants(tools[0]!)) { + expect(variant.additionalProperties).toBe(false); + expect(variant.required).toContain("type"); + } + }); } it("narrows the batch action schema when actions are provided", () => { const tools = openai.createComputerToolDefinitions({ actions: ["click"] }); expect(tools.map((tool) => tool.name)).toEqual([CUA_BATCH_TOOL_NAME]); - const actionsSchema = (tools[0]!.parameters as any).properties.actions.items; - const variants = actionsSchema.anyOf ?? [actionsSchema]; + const variants = batchActionVariants(tools[0]!); expect(variants).toHaveLength(1); expect(variants[0].properties.type.const).toBe("click"); expect(variants[0].properties.x).toBeTruthy(); expect(variants[0].properties.text).toBeUndefined(); }); + it("preserves action ordering in narrowed batch schemas", () => { + const subset: CuaActionType[] = ["screenshot", "type", "click"]; + const tools = openai.createComputerToolDefinitions({ actions: subset }); + const variants = batchActionVariants(tools[0]!); + expect(variants.map((v) => v.properties.type.const)).toEqual(subset); + }); + + it("emits a single-variant schema (not a union) when narrowed to one action", () => { + const tools = openai.createComputerToolDefinitions({ actions: ["click"] }); + const items = tools[0]!.parameters.properties.actions.items; + expect(items.anyOf).toBeUndefined(); + expect(items.oneOf).toBeUndefined(); + expect(items.properties.type.const).toBe("click"); + }); + + it("omits computer_use_extra navigation tool when actions are narrowed", () => { + const tools = openai.createComputerToolDefinitions({ actions: ["click", "goto"] }); + expect(tools).toHaveLength(1); + expect(tools[0]!.name).toBe(CUA_BATCH_TOOL_NAME); + }); + + it("exposes batch and navigation tool name constants identically across providers", () => { + for (const namespace of Object.values(providers)) { + const tools = namespace.createComputerToolDefinitions(); + expect(tools[0]!.name).toBe(CUA_BATCH_TOOL_NAME); + expect(tools[1]!.name).toBe(CUA_NAVIGATION_TOOL_NAME); + } + }); + it("exports provider coordinate systems", () => { expect(openai.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" }); expect(anthropic.COMPUTER_TOOL_COORDINATES).toEqual({ type: "pixel" });