Test and harden batch_computer_actions across providers

.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+      - run: npm ci
+      - run: npm run build --workspace @onkernel/cua-ai
+      - name: Unit tests
+        run: npm test --workspace @onkernel/cua-ai -- test/models.test.ts test/tools.test.ts
+
+  integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    # Only run on the main repo (not forks) so secrets are available.
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+      - run: npm ci
+      - run: npm run build --workspace @onkernel/cua-ai
+      - name: Integration tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }}
+          YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }}
+        run: npm test --workspace @onkernel/cua-ai -- test/batch-tool.integration.test.ts

packages/ai/src/providers/common.ts

@@ -245,6 +245,7 @@ const CUA_ACTION_SCHEMA_BY_TYPE = {
 
 export function createCuaActionSchema(actions: readonly CuaActionType[] = CUA_ACTION_TYPES): TSchema {
 	if (actions.length === 0) throw new Error("actions must include at least one CUA action type");
+	if (actions.length === 1) return CUA_ACTION_SCHEMA_BY_TYPE[actions[0]!];
 	return Type.Union(actions.map((action) => CUA_ACTION_SCHEMA_BY_TYPE[action]));
 }
 

packages/ai/src/providers/tzafon/provider.ts

@@ -47,6 +47,8 @@ export const streamTzafonResponses: StreamFunction<string, TzafonResponsesOption
 			if (options?.signal?.aborted) throw new Error("Request was aborted");
 
 			stream.push({ type: "start", partial: output });
+			output.responseId = getString(response, "id") || undefined;
+			output.usage = usageFromTzafon(getValue(response, "usage"));
 			for (const item of getArray(response, "output")) {
 				const type = getString(item, "type");
 				if (type === "message") {
@@ -195,16 +197,55 @@ function extractMessageText(item: unknown): string {
 }
 
 function parseArguments(value: unknown): Record<string, unknown> {
-	if (typeof value === "string" && value.trim()) {
-		try {
-			const parsed = JSON.parse(value);
-			return parsed && typeof parsed === "object" ? parsed as Record<string, unknown> : {};
-		} catch {
-			return {};
-		}
+	const top =
+		typeof value === "string" && value.trim()
+			? safeJsonParse(value)
+			: value && typeof value === "object"
+				? (value as Record<string, unknown>)
+				: {};
+	if (!top || typeof top !== "object") return {};
+	// Tzafon sometimes nests JSON-encoded arrays/objects inside the top-level argument object
+	// (observed: { "actions": "[{...}]" }). Unwrap one level so consumers get real values.
+	const out: Record<string, unknown> = {};
+	for (const [key, val] of Object.entries(top)) {
+		out[key] = typeof val === "string" && looksLikeJson(val) ? safeJsonParse(val) ?? val : val;
 	}
-	if (value && typeof value === "object") return value as Record<string, unknown>;
-	return {};
+	return out;
+}
+
+function safeJsonParse(value: string): Record<string, unknown> | unknown[] | null {
+	try {
+		const parsed = JSON.parse(value);
+		return parsed && typeof parsed === "object" ? parsed : null;
+	} catch {
+		return null;
+	}
+}
+
+function looksLikeJson(value: string): boolean {
+	const trimmed = value.trim();
+	return trimmed.startsWith("[") || trimmed.startsWith("{");
+}
+
+function usageFromTzafon(usage: unknown): AssistantMessage["usage"] {
+	const input = readNumber(usage, "input_tokens");
+	const output = readNumber(usage, "output_tokens");
+	const cacheRead = readNumber(getValue(usage, "input_tokens_details"), "cached_tokens");
+	const totalTokens = readNumber(usage, "total_tokens") || input + output;
+	return {
+		input,
+		output,
+		cacheRead,
+		cacheWrite: 0,
+		totalTokens,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+	};
+}
+
+function readNumber(obj: unknown, key: string): number {
+	if (!obj || typeof obj !== "object") return 0;
+	const n = (obj as Record<string, unknown>)[key];
+	return typeof n === "number" && Number.isFinite(n) ? n : 0;
 }
 
 function getArray(obj: unknown, key: string): unknown[] {

packages/ai/src/providers/yutori/provider.ts

@@ -13,7 +13,7 @@ import {
 	type TextContent,
 	type ToolCall,
 } from "@earendil-works/pi-ai";
-import { CUA_ACTION_TYPES } from "../common.js";
+import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, type CuaAction } from "../common.js";
 
 export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions";
 
@@ -92,21 +92,47 @@ async function runYutoriStream(
 		const text = typeof message?.content === "string" ? message.content : "";
 		if (text) emitText(stream, output, text);
 
+		const wantsBatch = (context.tools ?? []).some((tool) => tool.name === CUA_BATCH_TOOL_NAME);
+		const batchActions: CuaAction[] = [];
+		let firstBatchCallId: string | undefined;
+
 		for (const call of message?.tool_calls ?? []) {
 			if (call.type !== "function") continue;
+			const args = parseArguments(call.function.arguments);
+			const canonical = wantsBatch ? toCanonicalAction(call.function.name, args) : undefined;
+			if (canonical) {
+				batchActions.push(...canonical);
+				firstBatchCallId ??= call.id;
+				continue;
+			}
 			const contentIndex = output.content.length;
 			const toolCall: ToolCall = {
 				type: "toolCall",
 				id: call.id,
 				name: call.function.name,
-				arguments: parseArguments(call.function.arguments),
+				arguments: args,
 			};
 			output.content.push(toolCall);
 			stream.push({ type: "toolcall_start", contentIndex, partial: output });
 			stream.push({ type: "toolcall_delta", contentIndex, delta: call.function.arguments ?? "", partial: output });
 			stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output });
 		}
 
+		if (batchActions.length > 0) {
+			const contentIndex = output.content.length;
+			const toolCall: ToolCall = {
+				type: "toolCall",
+				id: firstBatchCallId ?? `yutori_batch_${Date.now()}`,
+				name: CUA_BATCH_TOOL_NAME,
+				arguments: { actions: batchActions },
+			};
+			output.content.push(toolCall);
+			output.stopReason = "toolUse";
+			stream.push({ type: "toolcall_start", contentIndex, partial: output });
+			stream.push({ type: "toolcall_delta", contentIndex, delta: JSON.stringify(toolCall.arguments), partial: output });
+			stream.push({ type: "toolcall_end", contentIndex, toolCall, partial: output });
+		}
+
 		stream.push({ type: "done", reason: output.stopReason as "stop" | "length" | "toolUse", message: output });
 		stream.end();
 	} catch (err) {
@@ -206,6 +232,72 @@ function parseArguments(value: string | undefined): Record<string, unknown> {
 	return JSON.parse(value) as Record<string, unknown>;
 }
 
+const SCROLL_AMOUNT_PER_NOTCH = 120;
+
+function readPoint(value: unknown): { x: number; y: number } | undefined {
+	if (!Array.isArray(value) || value.length < 2) return undefined;
+	const x = Number(value[0]);
+	const y = Number(value[1]);
+	if (!Number.isFinite(x) || !Number.isFinite(y)) return undefined;
+	return { x, y };
+}
+
+function toCanonicalAction(name: string, args: Record<string, unknown>): CuaAction[] | undefined {
+	const coords = readPoint(args.coordinates);
+	switch (name) {
+		case "left_click":
+			return coords ? [{ type: "click", x: coords.x, y: coords.y }] : undefined;
+		case "right_click":
+			return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "right" }] : undefined;
+		case "middle_click":
+			return coords ? [{ type: "click", x: coords.x, y: coords.y, button: "middle" }] : undefined;
+		case "double_click":
+			return coords ? [{ type: "double_click", x: coords.x, y: coords.y }] : undefined;
+		case "mouse_move":
+		case "hover":
+			return coords ? [{ type: "move", x: coords.x, y: coords.y }] : undefined;
+		case "mouse_down":
+			return coords ? [{ type: "mouse_down", x: coords.x, y: coords.y }] : undefined;
+		case "mouse_up":
+			return coords ? [{ type: "mouse_up", x: coords.x, y: coords.y }] : undefined;
+		case "type": {
+			const text = typeof args.text === "string" ? args.text : undefined;
+			return text !== undefined ? [{ type: "type", text }] : undefined;
+		}
+		case "key_press":
+		case "hold_key": {
+			const key = typeof args.key === "string" ? args.key : undefined;
+			return key ? [{ type: "keypress", keys: [key] }] : undefined;
+		}
+		case "scroll": {
+			if (!coords) return undefined;
+			const amount = typeof args.amount === "number" ? args.amount : 1;
+			const direction = typeof args.direction === "string" ? args.direction : "down";
+			const ticks = amount * SCROLL_AMOUNT_PER_NOTCH;
+			const dx = direction === "left" ? -ticks : direction === "right" ? ticks : 0;
+			const dy = direction === "up" ? -ticks : direction === "down" ? ticks : 0;
+			return [{ type: "scroll", x: coords.x, y: coords.y, scroll_x: dx, scroll_y: dy }];
+		}
+		case "drag": {
+			const start = readPoint(args.start_coordinates);
+			if (!start || !coords) return undefined;
+			return [{ type: "drag", path: [start, coords] }];
+		}
+		case "wait":
+			return [{ type: "wait" }];
+		case "go_back":
+			return [{ type: "back" }];
+		case "go_forward":
+			return [{ type: "forward" }];
+		case "goto_url": {
+			const url = typeof args.url === "string" ? args.url : undefined;
+			return url ? [{ type: "goto", url }] : undefined;
+		}
+		default:
+			return undefined;
+	}
+}
+
 function usageFromYutori(usage: unknown): AssistantMessage["usage"] {
 	const input = readNumber(usage, "prompt_tokens");
 	const output = readNumber(usage, "completion_tokens");