From 72565d2cae2ecca14448463e392f2d3c15ac7bae Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:06:27 +0000 Subject: [PATCH 1/2] Add OpenRouter GLM-V CUA support --- .github/workflows/ci.yml | 2 + packages/agent/README.md | 1 + packages/agent/test/e2e.live.test.ts | 15 ++++ packages/ai/README.md | 13 +-- packages/ai/docs/supported-models.md | 13 +++ packages/ai/examples/quickstart.ts | 1 + packages/ai/src/api-keys.ts | 1 + packages/ai/src/index.ts | 1 + packages/ai/src/models.ts | 11 ++- packages/ai/src/providers/openrouter/index.ts | 90 +++++++++++++++++++ packages/ai/src/runtime-spec.ts | 2 + packages/ai/test/api-keys.test.ts | 3 + .../ai/test/computer-tool.integration.test.ts | 40 +++++++-- packages/ai/test/models.test.ts | 16 +++- packages/ai/test/openrouter-provider.test.ts | 19 ++++ packages/ai/test/provider-module.test.ts | 4 +- packages/ai/test/providers.test.ts | 2 + packages/ai/test/runtime-spec.test.ts | 2 + packages/cli/README.md | 8 +- packages/cli/src/cli.ts | 3 + 20 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 packages/ai/src/providers/openrouter/index.ts create mode 100644 packages/ai/test/openrouter-provider.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 798e6e6..400e37f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm run test:integration --workspace @onkernel/cua-ai @@ -152,6 +153,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm test --workspace @onkernel/cua-agent -- test/e2e.live.test.ts diff --git a/packages/agent/README.md b/packages/agent/README.md index cb60d4f..8b9813c 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -105,6 +105,7 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` - Anthropic: `ANTHROPIC_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` - Gemini: `GOOGLE_API_KEY` or `GEMINI_API_KEY` +- OpenRouter: `OPENROUTER_API_KEY` - Tzafon: `TZAFON_API_KEY` - Yutori: `YUTORI_API_KEY` diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 41e3436..5d22068 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -20,6 +20,7 @@ type ProviderCase = { | "openai:gpt-5.5" | "anthropic:claude-opus-4-7" | "google:gemini-3-flash-preview" + | "openrouter:z-ai/glm-5v-turbo" | "tzafon:tzafon.northstar-cua-fast" | "yutori:n1.5-latest"; prompt: string; @@ -79,6 +80,20 @@ const cases: ProviderCase[] = [ timeoutMs: 300_000, ciOptInEnvVar: "CUA_E2E_GEMINI", }, + { + name: "openrouter", + apiKeyEnvVar: "OPENROUTER_API_KEY", + modelRef: "openrouter:z-ai/glm-5v-turbo", + prompt: [ + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", + "Do not call any other tools.", + "After the tool result, provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: true, + timeoutMs: 180_000, + ciOptInEnvVar: "CUA_E2E_OPENROUTER", + }, { name: "tzafon", apiKeyEnvVar: "TZAFON_API_KEY", diff --git a/packages/ai/README.md b/packages/ai/README.md index 6d0da7a..70ad0e1 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -20,6 +20,7 @@ check these environment variables, in order: | `openai` | `OPENAI_API_KEY` | | `anthropic` | `ANTHROPIC_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` | | `google` | `GOOGLE_API_KEY`, `GEMINI_API_KEY` | +| `openrouter` | `OPENROUTER_API_KEY` | | `tzafon` | `TZAFON_API_KEY` | | `yutori` | `YUTORI_API_KEY` | @@ -38,8 +39,8 @@ Pass the resolved key as the `apiKey` stream option (as in the Quick Start below) so a missing key fails loudly before any request is made. If you omit `apiKey`, pi-ai's built-in providers fall back to their own env lookup (`OPENAI_API_KEY`; `ANTHROPIC_OAUTH_TOKEN`/`ANTHROPIC_API_KEY`; for `google` -only `GEMINI_API_KEY`, not `GOOGLE_API_KEY`), and this package's Tzafon/Yutori -stream adapters read `TZAFON_API_KEY`/`YUTORI_API_KEY`. +only `GEMINI_API_KEY`, not `GOOGLE_API_KEY`; `OPENROUTER_API_KEY`), and this +package's Tzafon/Yutori stream adapters read `TZAFON_API_KEY`/`YUTORI_API_KEY`. ## Quick Start @@ -220,7 +221,7 @@ list of CUA-supporting models per provider. `CuaProvider` is the string union of provider IDs this package targets: ```ts -type CuaProvider = "openai" | "anthropic" | "google" | "tzafon" | "yutori"; +type CuaProvider = "openai" | "anthropic" | "google" | "openrouter" | "tzafon" | "yutori"; ``` The IDs match pi-ai's `Model.provider` values exactly. `providerForModel(model)` @@ -467,8 +468,8 @@ type CuaNavigationInput = { ## Provider Namespaces -Every provider namespace (`openai`, `anthropic`, `gemini`, `tzafon`, -`yutori`) follows one convention: +Every provider namespace (`openai`, `anthropic`, `gemini`, `openrouter`, +`tzafon`, `yutori`) follows one convention: - `computerTools(options?)` and `computerToolExecutors(options?)` - `createActionSchema(actions?)` — TypeBox schema for the provider's subset @@ -487,6 +488,8 @@ Provider-specific extras: - `openai`: `openaiResponsesStoreOnPayload` payload hook, plus the `computer_use_extra` navigation aliases `OPENAI_EXTRA_TOOL_NAME`, `OPENAI_EXTRA_TOOL_DESCRIPTION`, `OpenAIExtraSchema`, `OpenAIExtraInput` +- `openrouter`: OpenAI-compatible chat-completions transport for routed models + such as `openrouter:z-ai/glm-5v-turbo` and `openrouter:z-ai/glm-4.6v` - `anthropic`: `ANTHROPIC_BATCH_TOOL_NAME` (`"computer_batch"`) - `tzafon`: the `tzafon-responses` stream adapter (`TZAFON_RESPONSES_API`, `streamTzafonResponses`, `streamSimpleTzafonResponses`, diff --git a/packages/ai/docs/supported-models.md b/packages/ai/docs/supported-models.md index 1e0ae0c..b037099 100644 --- a/packages/ai/docs/supported-models.md +++ b/packages/ai/docs/supported-models.md @@ -57,6 +57,19 @@ Google's native `tools.computer_use` request wrapper instead. Source: [Gemini computer use docs](https://ai.google.dev/gemini-api/docs/computer-use). +## `openrouter` + +API: `openai-completions` · coordinates: pixel + +Exact IDs: + +- `z-ai/glm-5v-turbo` ([model page](https://openrouter.ai/z-ai/glm-5v-turbo)) +- `z-ai/glm-4.6v` ([model page](https://openrouter.ai/z-ai/glm-4.6v)) + +OpenRouter routes these as OpenAI-compatible chat-completions models. The +model-facing CUA tools are the canonical function tools exported by this +package, not a provider-native browser action schema. + ## `tzafon` API: `tzafon-responses` · coordinates: normalized 0–999 diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index 5c0e64f..fe02313 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -10,6 +10,7 @@ import { // Switch providers by setting CUA_MODEL (and the matching API key env var): // anthropic:claude-opus-4-7 ANTHROPIC_API_KEY // google:gemini-3-flash-preview GOOGLE_API_KEY +// openrouter:z-ai/glm-5v-turbo OPENROUTER_API_KEY // tzafon:tzafon.northstar-cua-fast TZAFON_API_KEY // yutori:n1.5-latest YUTORI_API_KEY const modelRef = (process.env.CUA_MODEL ?? "openai:gpt-5.5") as CuaModelRef; diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index cfbbed7..bc86478 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -12,6 +12,7 @@ const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { openai: ["OPENAI_API_KEY"], anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], + openrouter: ["OPENROUTER_API_KEY"], tzafon: ["TZAFON_API_KEY"], yutori: ["YUTORI_API_KEY"], }; diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 4bed13f..4f648b0 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -10,6 +10,7 @@ export * from "./providers/common"; export * as anthropic from "./providers/anthropic/index"; export * as gemini from "./providers/gemini/index"; export * as openai from "./providers/openai/index"; +export * as openrouter from "./providers/openrouter/index"; export * as tzafon from "./providers/tzafon/index"; export * as yutori from "./providers/yutori/index"; diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 1119cc0..199ef43 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -6,7 +6,7 @@ import { } from "@earendil-works/pi-ai"; /** Providers with curated computer-use model support. */ -export type CuaProvider = "openai" | "anthropic" | "google" | "tzafon" | "yutori"; +export type CuaProvider = "openai" | "anthropic" | "google" | "openrouter" | "tzafon" | "yutori"; /** Provider-qualified model reference, e.g. `"openai:gpt-5.5"` or `"google:gemini-3-flash-preview"`. */ export type CuaModelRef = `${CuaProvider}:${string}`; @@ -23,7 +23,7 @@ export interface CuaModelInfo { } /** All providers this package curates computer-use models for. */ -export const CUA_PROVIDERS: readonly CuaProvider[] = ["openai", "anthropic", "google", "tzafon", "yutori"]; +export const CUA_PROVIDERS: readonly CuaProvider[] = ["openai", "anthropic", "google", "openrouter", "tzafon", "yutori"]; /** * How a {@link CuaModelAnnotation} matches model ids. @@ -80,6 +80,10 @@ export const CUA_MODEL_ANNOTATIONS: Record[]> = { ], anthropic: [], google: [], + openrouter: [], tzafon: [ cuaModel("tzafon", "tzafon.northstar-cua-fast", "Tzafon Northstar CUA Fast"), cuaModel("tzafon", "tzafon.northstar-cua-fast-1.6", "Tzafon Northstar CUA Fast 1.6"), @@ -254,6 +259,8 @@ function cuaModel(provider: CuaProvider, id: string, name: string): Model { return { ...base, api: "anthropic-messages", baseUrl: "https://api.anthropic.com", contextWindow: 200_000, maxTokens: 64_000 } as Model; case "google": return { ...base, api: "google-generative-ai", baseUrl: "https://generativelanguage.googleapis.com/v1beta", contextWindow: 1_048_576, maxTokens: 65_536 } as Model; + case "openrouter": + return { ...base, api: "openai-completions", baseUrl: "https://openrouter.ai/api/v1", contextWindow: 128_000, maxTokens: 32_768 } as Model; case "tzafon": return { ...base, api: "tzafon-responses", baseUrl: "https://api.lightcone.ai", contextWindow: 128_000, maxTokens: 4_096 } as Model; case "yutori": diff --git a/packages/ai/src/providers/openrouter/index.ts b/packages/ai/src/providers/openrouter/index.ts new file mode 100644 index 0000000..23037e5 --- /dev/null +++ b/packages/ai/src/providers/openrouter/index.ts @@ -0,0 +1,90 @@ +import { + computerTools, + createCuaActionToolExecutors, + type ComputerToolsOptions, + type CuaAction, + type CuaToolExecutorSpec, +} from "../common"; +import type { ComputerToolCoordinateSystem, CuaProviderModule } from "../common"; + +export { + CUA_ACTION_TYPES as OPENROUTER_CUA_ACTION_TYPES, + computerTools, + createCuaActionSchema as createActionSchema, +} from "../common"; +export type { + ComputerToolsOptions, + CuaAction as OpenRouterAction, +} from "../common"; + +// OpenRouter exposes the GLM V models through its OpenAI-compatible Chat +// Completions API. The model-specific Z.AI docs describe function calling, but +// not a provider-native browser coordinate system, so CUA uses screenshot pixel +// coordinates with the standard canonical action tools. +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "pixel" }; +} + +export const OPENROUTER_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Use screenshot pixel coordinates for tool calls, and request screenshots or URL reads when state changes.`; + +export function buildOpenRouterSystemPrompt(opts: { suffix?: string } = {}): string { + return [OPENROUTER_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +} + +export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { + return createCuaActionToolExecutors(options.actions).map((executor) => ({ + ...executor, + toActions(args: unknown): CuaAction[] { + return executor.toActions(args).map(normalizeOpenRouterAction); + }, + })); +} + +export function normalizeOpenRouterAction(action: CuaAction): CuaAction { + // GLM-4.6V via OpenRouter can return [x, y] tuples in scalar coordinate + // fields even when the function schema asks for numbers. + switch (action.type) { + case "click": + case "double_click": + case "mouse_down": + case "mouse_up": + case "move": + return { + ...action, + x: coordinateNumber(action.x, 0) ?? action.x, + y: coordinateNumber(action.y, 1) ?? coordinateNumber(action.x, 1) ?? action.y, + }; + case "scroll": + return { + ...action, + x: coordinateNumber(action.x, 0) ?? action.x, + y: coordinateNumber(action.y, 1) ?? coordinateNumber(action.x, 1) ?? action.y, + }; + case "drag": + return { + ...action, + path: action.path.map((point) => ({ + x: coordinateNumber(point.x, 0) ?? point.x, + y: coordinateNumber(point.y, 1) ?? coordinateNumber(point.x, 1) ?? point.y, + })), + }; + default: + return action; + } +} + +function coordinateNumber(value: unknown, index: number): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (Array.isArray(value)) { + const next = value[index] ?? value[0]; + return typeof next === "number" && Number.isFinite(next) ? next : undefined; + } + return undefined; +} + +export const providerModule = { + toolDefinitions: computerTools, + toolExecutors: computerToolExecutors, + coordinateSystem, + buildSystemPrompt: buildOpenRouterSystemPrompt, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index fc9fdb5..2622822 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -3,6 +3,7 @@ import { getCuaModel, providerForModel } from "./models"; import { providerModule as anthropic } from "./providers/anthropic/index"; import { providerModule as gemini } from "./providers/gemini/index"; import { providerModule as openai } from "./providers/openai/index"; +import { providerModule as openrouter } from "./providers/openrouter/index"; import { providerModule as tzafon } from "./providers/tzafon/index"; import { providerModule as yutori } from "./providers/yutori/index"; import type { @@ -16,6 +17,7 @@ const PROVIDERS = { openai, anthropic, google: gemini, + openrouter, tzafon, yutori, } satisfies Record; diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index 501b010..34241db 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -12,6 +12,7 @@ const ENV_KEYS = [ "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", + "OPENROUTER_API_KEY", "TZAFON_API_KEY", "YUTORI_API_KEY", ] as const; @@ -29,12 +30,14 @@ afterEach(() => { describe("cua api key helpers", () => { it("maps provider names to expected environment variables", () => { expect(cuaApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("openrouter")).toEqual(["OPENROUTER_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("unknown")).toEqual([]); }); it("resolves provider api keys with fallback order", () => { + delete process.env.GOOGLE_API_KEY; process.env.GEMINI_API_KEY = "gemini"; expect(getCuaEnvApiKey("google")).toBe("gemini"); process.env.GOOGLE_API_KEY = "google"; diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 7eb5358..efd2b8d 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -6,12 +6,15 @@ import { CUA_ACTION_TYPES, type Context, type CuaActionType, + type CuaModelRef, type CuaProvider, anthropic, complete, gemini, getCuaModel, openai, + openrouter, + resolveCuaRuntimeSpec, tzafon, yutori, } from "../src/index"; @@ -22,7 +25,7 @@ const screenshotPath = join(here, "..", "examples", "screenshot.png"); interface ProviderCase { provider: CuaProvider; envVar: string; - modelRef: string; + modelRef: CuaModelRef; tools: () => ReturnType; coordinateRange: readonly [number, number]; requireToolCalls: boolean; @@ -56,6 +59,22 @@ const cases: ProviderCase[] = [ coordinateRange: [0, 999], requireToolCalls: true, }, + { + provider: "openrouter", + envVar: "OPENROUTER_API_KEY", + modelRef: "openrouter:z-ai/glm-5v-turbo", + tools: () => openrouter.computerTools({ actions: ["click"] }), + coordinateRange: [0, 1920], + requireToolCalls: true, + }, + { + provider: "openrouter", + envVar: "OPENROUTER_API_KEY", + modelRef: "openrouter:z-ai/glm-4.6v", + tools: () => openrouter.computerTools({ actions: ["click"] }), + coordinateRange: [0, 1920], + requireToolCalls: true, + }, { provider: "tzafon", envVar: "TZAFON_API_KEY", @@ -112,7 +131,7 @@ describe("individual computer action integration", () => { const test = hasKey ? it : it.skip; (ciEnabled ? test : it.skip)(`${c.provider} returns a canonical click tool call`, async () => { - const model = getCuaModel(c.modelRef as never); + const model = getCuaModel(c.modelRef); const context = await buildContext(c.tools); const response = await complete(model, context, { apiKey: process.env[c.envVar], @@ -131,13 +150,18 @@ describe("individual computer action integration", () => { const click = toolCalls.find((call) => call.name === "click"); expect(click, `${c.provider} did not return click; got [${toolCalls.map((call) => call.name).join(", ")}]`).toBeDefined(); - expect(typeof click!.arguments.x).toBe("number"); - expect(typeof click!.arguments.y).toBe("number"); + const executor = resolveCuaRuntimeSpec(c.modelRef).toolExecutors.find((candidate) => candidate.definition.name === "click"); + expect(executor, `${c.provider} has no click executor`).toBeDefined(); + const action = executor!.toActions(click!.arguments)[0]; + expect(action?.type).toBe("click"); + if (action?.type !== "click") throw new Error(`${c.provider} executor did not return a click action`); + expect(typeof action.x).toBe("number"); + expect(typeof action.y).toBe("number"); const [min, max] = c.coordinateRange; - expect(click!.arguments.x as number, `${c.provider} x out of range`).toBeGreaterThanOrEqual(min); - expect(click!.arguments.x as number).toBeLessThanOrEqual(max); - expect(click!.arguments.y as number, `${c.provider} y out of range`).toBeGreaterThanOrEqual(min); - expect(click!.arguments.y as number).toBeLessThanOrEqual(max); + expect(action.x, `${c.provider} x out of range`).toBeGreaterThanOrEqual(min); + expect(action.x).toBeLessThanOrEqual(max); + expect(action.y, `${c.provider} y out of range`).toBeGreaterThanOrEqual(min); + expect(action.y).toBeLessThanOrEqual(max); expect(response.usage.totalTokens, `${c.provider} usage tokens not reported`).toBeGreaterThan(0); }, 60_000); } diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index 2a6c3ad..9d071d5 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -23,7 +23,7 @@ describe("CUA model refs", () => { it("names the valid providers in the unsupported-provider error", () => { expect(() => parseCuaModelRef("bogus:model")).toThrow( - 'unsupported CUA provider "bogus" (expected one of: openai, anthropic, google, tzafon, yutori)', + 'unsupported CUA provider "bogus" (expected one of: openai, anthropic, google, openrouter, tzafon, yutori)', ); }); @@ -110,6 +110,20 @@ describe("CUA support annotations", () => { expect(findCuaAnnotation("yutori", "n1.5-latest")).toBeDefined(); expect(findCuaAnnotation("tzafon", "tzafon.northstar-cua-fast")).toBeDefined(); expect(findCuaAnnotation("tzafon", "tzafon.northstar-cua-fast-1.6")).toBeDefined(); + expect(findCuaAnnotation("openrouter", "z-ai/glm-5v-turbo")).toBeDefined(); + expect(findCuaAnnotation("openrouter", "z-ai/glm-4.6v")).toBeDefined(); + }); + + it("loads OpenRouter GLM-V models from pi-ai", () => { + const glm5v = getCuaModel("openrouter:z-ai/glm-5v-turbo"); + expect(glm5v.provider).toBe("openrouter"); + expect(glm5v.api).toBe("openai-completions"); + expect(glm5v.input).toContain("image"); + + const glm46v = getCuaModel("openrouter:z-ai/glm-4.6v"); + expect(glm46v.provider).toBe("openrouter"); + expect(glm46v.api).toBe("openai-completions"); + expect(glm46v.input).toContain("image"); }); it("no longer advertises the Gemini 2.5 computer-use preview", () => { diff --git a/packages/ai/test/openrouter-provider.test.ts b/packages/ai/test/openrouter-provider.test.ts new file mode 100644 index 0000000..85e1362 --- /dev/null +++ b/packages/ai/test/openrouter-provider.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { openrouter } from "../src/index"; + +describe("OpenRouter provider normalization", () => { + it("unwraps GLM-V coordinate arrays on canonical actions", () => { + const action = { + type: "click", + x: [929, 294], + y: [294, 294], + } as unknown as openrouter.OpenRouterAction; + + expect(openrouter.normalizeOpenRouterAction(action)).toEqual({ type: "click", x: 929, y: 294 }); + }); + + it("applies the coordinate normalization through tool executors", () => { + const click = openrouter.computerToolExecutors({ actions: ["click"] })[0]!; + expect(click.toActions({ x: [929, 294], y: [294, 294] })).toEqual([{ type: "click", x: 929, y: 294 }]); + }); +}); diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index 17b17a1..aff6047 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,11 +1,12 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import { anthropic, CUA_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, openrouter, tzafon, yutori } from "../src/index"; import type { CuaProviderModule } from "../src/providers/common"; const MODULES: Record = { openai, anthropic, google: gemini, + openrouter, tzafon, yutori, }; @@ -14,6 +15,7 @@ const NAMESPACES: Record; pref openai: { namespace: openai, prefix: "OPENAI" }, anthropic: { namespace: anthropic, prefix: "ANTHROPIC" }, google: { namespace: gemini, prefix: "GEMINI" }, + openrouter: { namespace: openrouter, prefix: "OPENROUTER" }, tzafon: { namespace: tzafon, prefix: "TZAFON" }, yutori: { namespace: yutori, prefix: "YUTORI" }, }; diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index 7804aff..a6b6c99 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -12,6 +12,7 @@ describe("CUA provider registration", () => { it("leaves pi-ai built-ins registered for the providers CUA targets", () => { expect(getApiProvider("openai-responses")).toBeDefined(); + expect(getApiProvider("openai-completions")).toBeDefined(); expect(getApiProvider("anthropic-messages")).toBeDefined(); expect(getApiProvider("google-generative-ai")).toBeDefined(); }); @@ -19,6 +20,7 @@ describe("CUA provider registration", () => { it("exposes a stream function on every CUA-target API", () => { for (const api of [ "openai-responses", + "openai-completions", "anthropic-messages", "google-generative-ai", YUTORI_CHAT_COMPLETIONS_API, diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index 8be6fcf..244007d 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -34,10 +34,12 @@ describe("resolveCuaRuntimeSpec", () => { it("only sets payload middleware for providers that need it", () => { const yutoriSpec = resolveCuaRuntimeSpec("yutori:n1.5-latest"); const openaiSpec = resolveCuaRuntimeSpec("openai:gpt-5.5"); + const openrouterSpec = resolveCuaRuntimeSpec("openrouter:z-ai/glm-5v-turbo"); const tzafonSpec = resolveCuaRuntimeSpec("tzafon:tzafon.northstar-cua-fast"); const anthropicSpec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7"); expect(yutoriSpec.onPayload).toBeTypeOf("function"); expect(openaiSpec.onPayload).toBeTypeOf("function"); + expect(openrouterSpec.onPayload).toBeUndefined(); expect(tzafonSpec.onPayload).toBeTypeOf("function"); expect(anthropicSpec.onPayload).toBeUndefined(); }); diff --git a/packages/cli/README.md b/packages/cli/README.md index 2e12229..20a5f13 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -45,6 +45,8 @@ cua models -p openai cua --print --model openai:gpt-5.5 "..." cua --print --model anthropic:claude-opus-4-7 "..." cua --print --model google:gemini-3-flash-preview "..." +cua --print --model openrouter:z-ai/glm-5v-turbo "..." +cua --print --model tzafon:tzafon.northstar-cua-fast "..." cua --print --model yutori:n1.5-latest "..." # Named sessions (browser stays alive across calls): @@ -67,8 +69,8 @@ cua --session abc12345 # by id prefix Run `cua models` to list every supported `-m` / `--model` value and the provider it routes to. Filter by provider with `cua models -p openai`, -`cua models -p anthropic`, `cua models -p google` (alias: `gemini`), or -`cua models -p yutori`. +`cua models -p anthropic`, `cua models -p google` (alias: `gemini`), +`cua models -p openrouter`, `cua models -p tzafon`, or `cua models -p yutori`. `-m` / `--model` accepts a provider-qualified `provider:model` ref (e.g. `openai:gpt-5.5`) or a bare model id when it matches exactly one catalog @@ -85,12 +87,14 @@ Configuration is by environment variable. There is no config file. | `ANTHROPIC_API_KEY` | Anthropic API key (required when `-m anthropic:…`) | | `GOOGLE_API_KEY` | Google API key (required when `-m google:…`) | | `GEMINI_API_KEY` | alias of `GOOGLE_API_KEY` | +| `OPENROUTER_API_KEY` | OpenRouter API key (required when `-m openrouter:…`) | | `TZAFON_API_KEY` | Tzafon API key (required when `-m tzafon:…`) | | `YUTORI_API_KEY` | Yutori API key (required when `-m yutori:…`) | | `KERNEL_BASE_URL` | override Kernel base URL | | `OPENAI_BASE_URL` | override OpenAI base URL | | `ANTHROPIC_BASE_URL` | override Anthropic base URL | | `GOOGLE_BASE_URL` | override Google base URL | +| `OPENROUTER_BASE_URL` | override OpenRouter base URL | | `TZAFON_BASE_URL` | override Tzafon base URL | | `YUTORI_BASE_URL` | override Yutori base URL | | `XDG_DATA_HOME` | sessions dir base (defaults to `~/.local/share`) | diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 5c20070..df379d0 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -37,6 +37,7 @@ Options: openai: openai:gpt-5.5 anthropic: anthropic:claude-opus-4-7 google: google:gemini-3-flash-preview + openrouter: openrouter:z-ai/glm-5v-turbo tzafon: tzafon:tzafon.northstar-cua-fast yutori: yutori:n1.5-latest --thinking Thinking level: off | minimal | low | medium | high | xhigh @@ -75,12 +76,14 @@ Environment: ANTHROPIC_API_KEY Anthropic API key (required when -m anthropic:…) GOOGLE_API_KEY Google API key (required when -m google:…) GEMINI_API_KEY Alias for GOOGLE_API_KEY + OPENROUTER_API_KEY OpenRouter API key (required when -m openrouter:…) TZAFON_API_KEY Tzafon API key (required when -m tzafon:…) YUTORI_API_KEY Yutori API key (required when -m yutori:…) KERNEL_BASE_URL Override Kernel base URL OPENAI_BASE_URL Override OpenAI base URL ANTHROPIC_BASE_URL Override Anthropic base URL GOOGLE_BASE_URL Override Google base URL + OPENROUTER_BASE_URL Override OpenRouter base URL TZAFON_BASE_URL Override Tzafon base URL YUTORI_BASE_URL Override Yutori base URL XDG_DATA_HOME Sessions are stored under \$XDG_DATA_HOME/cua/sessions From faf93f5a11790b5c9d457c692eab28e64c6f91af Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:20:20 +0000 Subject: [PATCH 2/2] Fix OpenRouter coordinate normalization --- packages/ai/src/providers/openrouter/index.ts | 54 ++++++++++++------- .../ai/test/computer-tool.integration.test.ts | 2 +- packages/ai/test/openrouter-provider.test.ts | 14 +++++ 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/packages/ai/src/providers/openrouter/index.ts b/packages/ai/src/providers/openrouter/index.ts index 23037e5..9ae7f3c 100644 --- a/packages/ai/src/providers/openrouter/index.ts +++ b/packages/ai/src/providers/openrouter/index.ts @@ -31,6 +31,7 @@ export function buildOpenRouterSystemPrompt(opts: { suffix?: string } = {}): str return [OPENROUTER_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } +/** Build OpenRouter execution adapters and normalize GLM tuple-shaped coordinates before browser execution. */ export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaToolExecutorSpec[] { return createCuaActionToolExecutors(options.actions).map((executor) => ({ ...executor, @@ -40,39 +41,50 @@ export function computerToolExecutors(options: ComputerToolsOptions = {}): CuaTo })); } +/** Normalize OpenRouter GLM-V coordinate arguments into canonical CUA numeric coordinates. */ export function normalizeOpenRouterAction(action: CuaAction): CuaAction { - // GLM-4.6V via OpenRouter can return [x, y] tuples in scalar coordinate - // fields even when the function schema asks for numbers. switch (action.type) { case "click": case "double_click": case "mouse_down": case "mouse_up": - case "move": - return { - ...action, - x: coordinateNumber(action.x, 0) ?? action.x, - y: coordinateNumber(action.y, 1) ?? coordinateNumber(action.x, 1) ?? action.y, - }; - case "scroll": - return { - ...action, - x: coordinateNumber(action.x, 0) ?? action.x, - y: coordinateNumber(action.y, 1) ?? coordinateNumber(action.x, 1) ?? action.y, - }; + case "move": { + const { x, y, ...rest } = action; + return { ...rest, ...normalizeRequiredPoint(action.type, x, y) }; + } + case "scroll": { + const { x, y, ...rest } = action; + return { ...rest, ...normalizeOptionalPoint(action.type, x, y) }; + } case "drag": return { ...action, - path: action.path.map((point) => ({ - x: coordinateNumber(point.x, 0) ?? point.x, - y: coordinateNumber(point.y, 1) ?? coordinateNumber(point.x, 1) ?? point.y, - })), + path: action.path.map((point) => normalizeRequiredPoint("drag path", point.x, point.y)), }; default: return action; } } +function normalizeRequiredPoint(label: string, xValue: unknown, yValue: unknown): { x: number; y: number } { + const x = coordinateNumber(xValue, 0); + const y = coordinateNumber(yValue, 1) ?? coordinateTupleNumber(xValue, 1); + if (x === undefined || y === undefined) throw new Error(`invalid OpenRouter ${label} coordinates`); + return { x, y }; +} + +function normalizeOptionalPoint(label: string, xValue: unknown, yValue: unknown): { x?: number; y?: number } { + const x = coordinateNumber(xValue, 0); + const yFromTuple = coordinateTupleNumber(xValue, 1); + const y = coordinateNumber(yValue, 1) ?? yFromTuple; + if (xValue !== undefined && x === undefined) throw new Error(`invalid OpenRouter ${label} x coordinate`); + if ((yValue !== undefined || yFromTuple !== undefined) && y === undefined) throw new Error(`invalid OpenRouter ${label} y coordinate`); + return { + ...(x !== undefined ? { x } : {}), + ...(y !== undefined ? { y } : {}), + }; +} + function coordinateNumber(value: unknown, index: number): number | undefined { if (typeof value === "number" && Number.isFinite(value)) return value; if (Array.isArray(value)) { @@ -82,6 +94,12 @@ function coordinateNumber(value: unknown, index: number): number | undefined { return undefined; } +function coordinateTupleNumber(value: unknown, index: number): number | undefined { + if (!Array.isArray(value)) return undefined; + const next = value[index]; + return typeof next === "number" && Number.isFinite(next) ? next : undefined; +} + export const providerModule = { toolDefinitions: computerTools, toolExecutors: computerToolExecutors, diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index efd2b8d..9e9b3b4 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -130,7 +130,7 @@ describe("individual computer action integration", () => { const ciEnabled = !c.ciOptInEnvVar || !process.env.CI || process.env[c.ciOptInEnvVar] === "1"; const test = hasKey ? it : it.skip; - (ciEnabled ? test : it.skip)(`${c.provider} returns a canonical click tool call`, async () => { + (ciEnabled ? test : it.skip)(`${c.modelRef} returns a canonical click tool call`, async () => { const model = getCuaModel(c.modelRef); const context = await buildContext(c.tools); const response = await complete(model, context, { diff --git a/packages/ai/test/openrouter-provider.test.ts b/packages/ai/test/openrouter-provider.test.ts index 85e1362..5e9421b 100644 --- a/packages/ai/test/openrouter-provider.test.ts +++ b/packages/ai/test/openrouter-provider.test.ts @@ -12,8 +12,22 @@ describe("OpenRouter provider normalization", () => { expect(openrouter.normalizeOpenRouterAction(action)).toEqual({ type: "click", x: 929, y: 294 }); }); + it("does not copy scalar x into missing y coordinates", () => { + const action = { + type: "click", + x: 929, + } as unknown as openrouter.OpenRouterAction; + + expect(() => openrouter.normalizeOpenRouterAction(action)).toThrow("invalid OpenRouter click coordinates"); + }); + it("applies the coordinate normalization through tool executors", () => { const click = openrouter.computerToolExecutors({ actions: ["click"] })[0]!; expect(click.toActions({ x: [929, 294], y: [294, 294] })).toEqual([{ type: "click", x: 929, y: 294 }]); }); + + it("rejects unparseable coordinate arrays before execution", () => { + const click = openrouter.computerToolExecutors({ actions: ["click"] })[0]!; + expect(() => click.toActions({ x: ["929"], y: ["294"] })).toThrow("invalid OpenRouter click coordinates"); + }); });