diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fee0e0d..b140bd14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 # Only run on the main repo (not forks) so secrets are available. - if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' + # Skip draft PRs; still run on every commit to non-draft PRs and on main pushes. + if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft) steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -45,3 +46,29 @@ jobs: TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm test --workspace @onkernel/cua-ai -- test/batch-tool.integration.test.ts + + agent-e2e: + runs-on: ubuntu-latest + timeout-minutes: 45 + # Only run on the main repo (not forks) so secrets are available. + # Skip draft PRs; still run on every commit to non-draft PRs and on main pushes. + if: github.event_name == 'push' || (github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - run: npm run build --workspace @onkernel/cua-agent + - name: Agent live smoke tests (all providers) + env: + CUA_E2E_LIVE: "1" + KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} + YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} + run: npm test --workspace @onkernel/cua-agent -- test/e2e.live.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 61726764..328cd292 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,6 +5,9 @@ someone who wants to read the code, contribute, or fork. ## Design goals and invariants +- `@onkernel/cua-agent` is provider-neutral runtime glue around + `pi-agent-core`. It should never branch on provider identity directly. + Provider-specific behavior must come from `@onkernel/cua-ai`. - Provider packages are generic glue between a model provider and Kernel browsers. Their package roots expose provider-neutral helpers, tool specs, and execution functions that can be embedded in non-`pi` loops. @@ -28,6 +31,25 @@ someone who wants to read the code, contribute, or fork. and TUI regression tests. It is part of the monorepo build graph, but not part of the runtime browser/model/provider path. +## `cua-ai` vs `cua-agent` ownership boundary + +`@onkernel/cua-ai` and `@onkernel/cua-agent` intentionally split concerns: + +- `@onkernel/cua-ai` owns provider-specific policy: + - provider model refs and provider resolution + - provider default system prompts + - provider payload transforms and protocol quirks (for example, Yutori tool serialization policy) + - canonical CUA tool-definition exports +- `@onkernel/cua-agent` owns browser execution orchestration: + - `CuaAgent` / `CuaHarness` class wiring around `pi-agent-core` + - executing canonical CUA tool calls against Kernel browsers + - typed executor coverage and translator integration + +In practice this means any new provider quirk should be implemented in +`@onkernel/cua-ai` and surfaced through provider-neutral runtime specs. +`@onkernel/cua-agent` should consume that spec without explicit +provider-specific conditionals. + ## Layers `cua` is a thin TypeScript monorepo on top of the diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 98a75cd7..a970cf27 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,6 @@ ## 0.1.0 -- Initial `@onkernel/cua-agent` package. -- Added `createCuaAgent()` returning a pi-agent-core `Agent`. -- Added Kernel browser computer `AgentTool` constructors with internal translator plumbing. +- Class-first CUA runtime: `CuaAgent` and `CuaHarness` on top of pi-agent-core. +- Provider-neutral browser tool executors for canonical CUA tool names, backed by Kernel browser actions. +- Includes examples plus unit and live e2e coverage for common provider/model combinations. diff --git a/packages/agent/README.md b/packages/agent/README.md index 3e91108f..d7efbf9c 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -1,11 +1,10 @@ # `@onkernel/cua-agent` -Kernel browser computer-use tools and a small `Agent` factory built on +Kernel browser computer-use classes built on [`@earendil-works/pi-agent-core`](https://github.com/earendil-works/pi/tree/main/packages/agent). -The package keeps pi-agent-core's `Agent`, `AgentOptions`, `AgentTool`, event -stream, and state model intact. Kernel only supplies the browser execution -plumbing. +This package keeps pi-agent-core semantics intact and adds browser execution +plumbing for canonical CUA tools. ## Installation @@ -13,16 +12,16 @@ plumbing. npm install @onkernel/cua-agent @onkernel/cua-ai @onkernel/sdk ``` -## Quick Start +## Quick Start (`CuaAgent`) ```ts import Kernel from "@onkernel/sdk"; -import { createCuaAgent } from "@onkernel/cua-agent"; +import { CuaAgent } from "@onkernel/cua-agent"; const client = new Kernel({ apiKey: process.env.KERNEL_API_KEY! }); const browser = await client.browsers.create({ stealth: true }); -const agent = createCuaAgent({ +const agent = new CuaAgent({ browser, client, initialState: { @@ -34,52 +33,71 @@ const agent = createCuaAgent({ await agent.prompt("Open news.ycombinator.com and summarize the top story."); ``` -## Core Concepts - -### It Returns a pi Agent - -`createCuaAgent()` returns the underlying pi-agent-core `Agent` directly. -Subscribe to events, mutate `agent.state`, call `prompt()`, `continue()`, -`steer()`, and `followUp()` the same way you would with pi-agent-core. - -### Configuration Lives in `initialState` - -The API mirrors the pi-agent-core quick start: +## Quick Start (`CuaHarness`) ```ts -const agent = createCuaAgent({ +import { CuaHarness } from "@onkernel/cua-agent"; + +const harness = new CuaHarness({ browser, client, - initialState: { - model: "yutori:n1.5-latest", - tools: myTools, - systemPrompt: "Use the browser to complete the task.", - }, + model: "openai:gpt-5.5", }); + +await harness.prompt("Open example.com and tell me the current URL."); +const transcript = harness.getTranscript(); +console.log("messages in transcript:", transcript.length); ``` -If `initialState.tools` is omitted, Kernel installs the provider-specific CUA -computer tools. If `initialState.tools` is provided, it is used exactly. +Use `CuaAgent` when you want direct pi `Agent` control. Use `CuaHarness` when you +want the harness-style constructor plus transcript-oriented helpers like +`getTranscript()`. + +## Core Concepts + +### Class-First API + +- `CuaAgent extends Agent` +- `CuaHarness` wraps a pi `Agent` with a harness-style constructor and + delegated runtime methods. + +Both classes mirror pi constructor shapes and behavior, with minimal additions: +- `browser` (Kernel browser response) +- `client` (Kernel SDK client) +- CUA model refs (`"provider:model"`) accepted where pi expects a concrete model + +If `getApiKey` is omitted, both classes default to CUA env var conventions: +- OpenAI: `OPENAI_API_KEY` +- Anthropic: `ANTHROPIC_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` +- Gemini: `GOOGLE_API_KEY` or `GEMINI_API_KEY` +- Tzafon: `TZAFON_API_KEY` +- Yutori: `YUTORI_API_KEY` + +### Tool Defaults + +If tools are omitted, the classes install canonical CUA computer tool executors +using runtime specs from `@onkernel/cua-ai`. If tools are provided, they are +used exactly. ### Tool Composition -Use `createCuaComputerTools()` when you want to extend the default set: +Use `createCuaComputerTools()` to compose your own tool list from canonical +tool definitions: ```ts +import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; +import { createCuaComputerTools } from "@onkernel/cua-agent"; + +const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tools = [ - ...createCuaComputerTools({ provider: "openai", browser, client }), + ...createCuaComputerTools({ + browser, + client, + toolDefinitions: runtime.toolDefinitions, + }), myCustomTool, ]; ``` -There are no magic tool preset strings and no bundled coding/file tools. Compose -those yourself with pi packages or your own tools. - -### Browser Plumbing - -Public helpers accept Kernel SDK browser responses plus a Kernel client. The -internal translator handles screenshots, coordinate conversion, URL reads, and -Kernel computer API calls behind the scenes. - For full event semantics, steering, follow-up queues, and tool execution details, see the pi-agent-core README. diff --git a/packages/agent/examples/agent-openai-smoke.ts b/packages/agent/examples/agent-openai-smoke.ts new file mode 100644 index 00000000..38a09cbb --- /dev/null +++ b/packages/agent/examples/agent-openai-smoke.ts @@ -0,0 +1,41 @@ +import Kernel from "@onkernel/sdk"; +import { requireCuaEnvApiKeyForModel, type CuaModelRef } from "@onkernel/cua-ai"; +import { CuaAgent } from "../src/index"; +import { SCENARIOS } from "./shared/scenarios"; + +const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "openai:gpt-5.5"; + +async function main(): Promise { + const kernelApiKey = process.env.KERNEL_API_KEY; + if (!kernelApiKey) throw new Error("KERNEL_API_KEY is required"); + requireCuaEnvApiKeyForModel(modelRef); + const client = new Kernel({ apiKey: kernelApiKey }); + const browser = await client.browsers.create({ stealth: true }); + + try { + const agent = new CuaAgent({ + browser, + client, + initialState: { model: modelRef }, + }); + + agent.subscribe((event) => { + if (event.type === "tool_execution_start") { + console.log(`[tool:start] ${event.toolName}`); + } + if (event.type === "tool_execution_end") { + console.log(`[tool:end] ${event.toolName} error=${event.isError}`); + } + }); + + const scenario = SCENARIOS[0]!; + console.log(`running scenario: ${scenario.name}`); + await agent.prompt(scenario.prompt); + const assistant = [...agent.state.messages].reverse().find((message) => message.role === "assistant"); + console.log("assistant stopReason:", assistant?.role === "assistant" ? assistant.stopReason : "unknown"); + } finally { + await client.browsers.deleteByID(browser.session_id); + } +} + +void main(); diff --git a/packages/agent/examples/agent-provider-matrix.ts b/packages/agent/examples/agent-provider-matrix.ts new file mode 100644 index 00000000..e661d4af --- /dev/null +++ b/packages/agent/examples/agent-provider-matrix.ts @@ -0,0 +1,30 @@ +import Kernel from "@onkernel/sdk"; +import { requireCuaEnvApiKeyForModel, type CuaModelRef } from "@onkernel/cua-ai"; +import { CuaAgent } from "../src/index"; +import { SCENARIOS } from "./shared/scenarios"; + +const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "openai:gpt-5.5"; +const scenarioName = process.env.SCENARIO ?? SCENARIOS[0]!.name; + +async function main(): Promise { + const kernelApiKey = process.env.KERNEL_API_KEY; + if (!kernelApiKey) throw new Error("KERNEL_API_KEY is required"); + requireCuaEnvApiKeyForModel(modelRef); + const client = new Kernel({ apiKey: kernelApiKey }); + const browser = await client.browsers.create({ stealth: true }); + const scenario = SCENARIOS.find((entry) => entry.name === scenarioName) ?? SCENARIOS[0]!; + + try { + const agent = new CuaAgent({ + browser, + client, + initialState: { model: modelRef }, + }); + console.log(`model=${modelRef} scenario=${scenario.name}`); + await agent.prompt(scenario.prompt); + } finally { + await client.browsers.deleteByID(browser.session_id); + } +} + +void main(); diff --git a/packages/agent/examples/harness-openai-smoke.ts b/packages/agent/examples/harness-openai-smoke.ts new file mode 100644 index 00000000..3c38edcc --- /dev/null +++ b/packages/agent/examples/harness-openai-smoke.ts @@ -0,0 +1,43 @@ +import Kernel from "@onkernel/sdk"; +import { requireCuaEnvApiKeyForModel, type CuaModelRef } from "@onkernel/cua-ai"; +import { CuaHarness } from "../src/index"; +import { SCENARIOS } from "./shared/scenarios"; + +const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "openai:gpt-5.5"; + +async function main(): Promise { + const kernelApiKey = process.env.KERNEL_API_KEY; + if (!kernelApiKey) throw new Error("KERNEL_API_KEY is required"); + requireCuaEnvApiKeyForModel(modelRef); + const client = new Kernel({ apiKey: kernelApiKey }); + const browser = await client.browsers.create({ stealth: true }); + + try { + const harness = new CuaHarness({ + browser, + client, + model: modelRef, + }); + + harness.subscribe((event) => { + if (event.type === "tool_execution_start") { + console.log(`[tool:start] ${event.toolName}`); + } + if (event.type === "tool_execution_end") { + console.log(`[tool:end] ${event.toolName} error=${event.isError}`); + } + }); + + const scenario = SCENARIOS[0]!; + console.log(`running scenario: ${scenario.name}`); + await harness.prompt(scenario.prompt); + const transcript = harness.getTranscript(); + const lastAssistant = [...transcript].reverse().find((message) => message.role === "assistant"); + console.log("transcript messages:", transcript.length); + console.log("assistant stopReason:", lastAssistant?.role === "assistant" ? lastAssistant.stopReason : "unknown"); + } finally { + await client.browsers.deleteByID(browser.session_id); + } +} + +void main(); diff --git a/packages/agent/examples/harness-provider-matrix.ts b/packages/agent/examples/harness-provider-matrix.ts new file mode 100644 index 00000000..6970632b --- /dev/null +++ b/packages/agent/examples/harness-provider-matrix.ts @@ -0,0 +1,31 @@ +import Kernel from "@onkernel/sdk"; +import { requireCuaEnvApiKeyForModel, type CuaModelRef } from "@onkernel/cua-ai"; +import { CuaHarness } from "../src/index"; +import { SCENARIOS } from "./shared/scenarios"; + +const modelRef = (process.env.MODEL_REF as CuaModelRef | undefined) ?? "openai:gpt-5.5"; +const scenarioName = process.env.SCENARIO ?? SCENARIOS[0]!.name; + +async function main(): Promise { + const kernelApiKey = process.env.KERNEL_API_KEY; + if (!kernelApiKey) throw new Error("KERNEL_API_KEY is required"); + requireCuaEnvApiKeyForModel(modelRef); + const client = new Kernel({ apiKey: kernelApiKey }); + const browser = await client.browsers.create({ stealth: true }); + const scenario = SCENARIOS.find((entry) => entry.name === scenarioName) ?? SCENARIOS[0]!; + + try { + const harness = new CuaHarness({ + browser, + client, + model: modelRef, + }); + console.log(`model=${modelRef} scenario=${scenario.name}`); + await harness.prompt(scenario.prompt); + console.log("transcript messages:", harness.getTranscript().length); + } finally { + await client.browsers.deleteByID(browser.session_id); + } +} + +void main(); diff --git a/packages/agent/examples/shared/scenarios.ts b/packages/agent/examples/shared/scenarios.ts new file mode 100644 index 00000000..7626dd30 --- /dev/null +++ b/packages/agent/examples/shared/scenarios.ts @@ -0,0 +1,39 @@ +export interface BrowserScenario { + name: string; + prompt: string; +} + +export const SCENARIOS: BrowserScenario[] = [ + { + name: "example-link-and-url", + prompt: [ + "Use browser tools for exactly these steps:", + "1) goto https://example.com", + "2) read current url", + "3) click the More information link", + "4) read current url", + "5) capture a screenshot", + ].join("\n"), + }, + { + name: "hn-url-and-screenshot", + prompt: [ + "Use browser tools for exactly these steps:", + "1) goto https://news.ycombinator.com", + "2) scroll slightly", + "3) read current url", + "4) capture a screenshot", + ].join("\n"), + }, + { + name: "wikipedia-search", + prompt: [ + "Use browser tools for exactly these steps:", + "1) goto https://www.wikipedia.org", + "2) click the search input", + "3) type kernel", + "4) press Enter", + "5) read current url", + ].join("\n"), + }, +]; diff --git a/packages/agent/package.json b/packages/agent/package.json index 84b177cd..cf11139c 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,24 +1,28 @@ { "name": "@onkernel/cua-agent", "version": "0.1.0", - "description": "Kernel browser computer-use AgentTools and agent factory built on pi-agent-core", + "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", + "source": "./src/index.ts", "import": "./dist/index.js" } }, "files": [ "dist", + "examples", "README.md", "CHANGELOG.md" ], "scripts": { "build": "tsc -b", "clean": "tsc -b --clean", + "example:agent": "NODE_OPTIONS=--conditions=source tsx examples/agent-openai-smoke.ts", + "example:harness": "NODE_OPTIONS=--conditions=source tsx examples/harness-openai-smoke.ts", "test": "vitest --run" }, "dependencies": { diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index c5edc0e7..8120855f 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -2,83 +2,159 @@ import { Agent, type AgentOptions, type AgentTool, + type AgentEvent, + type AgentMessage, } from "@earendil-works/pi-agent-core"; import { + type ImageContent, type Api, type Model, - type SimpleStreamOptions, type CuaModelRef, - getCuaModel, - providerForModel, + getCuaEnvApiKey, + resolveCuaRuntimeSpec, streamSimple, - anthropic, - gemini, - openai, - tzafon, - yutori, } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; -import { createCuaComputerTools } from "./tools.js"; -import type { KernelBrowser } from "./translator/translator.js"; +import { createCuaComputerTools } from "./tools"; +import type { KernelBrowser } from "./translator/translator"; -export type CreateCuaAgentOptions = Omit & { +type CuaAgentInitialState = Omit, "model" | "tools"> & { + model: CuaModelRef | Model; + tools?: AgentTool[]; +}; + +export type CuaAgentOptions = Omit & { browser: KernelBrowser; - client?: Kernel; - initialState: Omit, "model" | "tools"> & { - model: CuaModelRef | Model; - tools?: AgentTool[]; - }; + client: Kernel; + initialState: CuaAgentInitialState; }; -export function createCuaAgent(options: CreateCuaAgentOptions): Agent { - const model = typeof options.initialState.model === "string" ? getCuaModel(options.initialState.model) : options.initialState.model; - const provider = providerForModel(model); - const tools = options.initialState.tools ?? createCuaComputerTools({ provider, browser: options.browser, client: options.client }); - const systemPrompt = options.initialState.systemPrompt ?? defaultSystemPrompt(provider); - const onPayload = composeOnPayload(defaultOnPayload(provider), options.onPayload); - - return new Agent({ - ...options, - streamFn: options.streamFn ?? streamSimple, - onPayload, - initialState: { - ...options.initialState, - model, - tools, - systemPrompt, - }, - }); -} +export type CuaHarnessOptions = Omit & { + browser: KernelBrowser; + client: Kernel; + model: CuaModelRef | Model; + tools?: AgentTool[]; + systemPrompt?: string; +}; -function defaultSystemPrompt(provider: ReturnType): string { - switch (provider) { - case "anthropic": - return anthropic.buildAnthropicSystemPrompt(); - case "google": - return gemini.buildGeminiSystemPrompt(); - case "tzafon": - return tzafon.buildTzafonSystemPrompt(); - case "yutori": - return yutori.buildYutoriSystemPrompt(); - case "openai": - default: - return openai.OPENAI_BATCH_INSTRUCTIONS; +export class CuaAgent extends Agent { + constructor(options: CuaAgentOptions) { + const { browser, client, initialState, onPayload, streamFn, ...agentOptions } = options; + const runtimeSpec = resolveCuaRuntimeSpec(initialState.model); + const tools = initialState.tools ?? createCuaComputerTools({ browser, client, toolDefinitions: runtimeSpec.toolDefinitions }); + const systemPrompt = initialState.systemPrompt ?? runtimeSpec.defaultSystemPrompt; + + super({ + ...agentOptions, + getApiKey: agentOptions.getApiKey ?? getCuaEnvApiKey, + streamFn: streamFn ?? streamSimple, + onPayload: composeOnPayload(runtimeSpec.onPayload, onPayload), + initialState: { + ...initialState, + model: runtimeSpec.model, + tools, + systemPrompt, + }, + }); } } -function defaultOnPayload(provider: ReturnType): SimpleStreamOptions["onPayload"] | undefined { - if (provider === "yutori") return yutori.yutoriBuiltinToolsOnPayload; - return undefined; +export class CuaHarness { + readonly agent: Agent; + + constructor(options: CuaHarnessOptions) { + const { browser, client, model, tools, systemPrompt, onPayload, streamFn, ...agentOptions } = options; + const runtimeSpec = resolveCuaRuntimeSpec(model); + const resolvedTools = tools ?? createCuaComputerTools({ browser, client, toolDefinitions: runtimeSpec.toolDefinitions }); + this.agent = new Agent({ + ...agentOptions, + getApiKey: agentOptions.getApiKey ?? getCuaEnvApiKey, + streamFn: streamFn ?? streamSimple, + onPayload: composeOnPayload(runtimeSpec.onPayload, onPayload), + initialState: { + model: runtimeSpec.model, + tools: resolvedTools, + systemPrompt: systemPrompt ?? runtimeSpec.defaultSystemPrompt, + }, + }); + } + + get state() { + return this.agent.state; + } + + get steeringMode() { + return this.agent.steeringMode; + } + + set steeringMode(mode: "all" | "one-at-a-time") { + this.agent.steeringMode = mode; + } + + get followUpMode() { + return this.agent.followUpMode; + } + + set followUpMode(mode: "all" | "one-at-a-time") { + this.agent.followUpMode = mode; + } + + subscribe(listener: (event: AgentEvent, signal: AbortSignal) => Promise | void): () => void { + return this.agent.subscribe(listener); + } + + async prompt(message: AgentMessage | AgentMessage[]): Promise; + async prompt(input: string, images?: ImageContent[]): Promise; + async prompt(input: string | AgentMessage | AgentMessage[], images?: ImageContent[]): Promise { + if (typeof input === "string") { + await this.agent.prompt(input, images); + return; + } + await this.agent.prompt(input); + } + + steer(message: AgentMessage): void { + this.agent.steer(message); + } + + followUp(message: AgentMessage): void { + this.agent.followUp(message); + } + + async continue(): Promise { + await this.agent.continue(); + } + + clearSteeringQueue(): void { + this.agent.clearSteeringQueue(); + } + + clearFollowUpQueue(): void { + this.agent.clearFollowUpQueue(); + } + + clearAllQueues(): void { + this.agent.clearAllQueues(); + } + + abort(): void { + this.agent.abort(); + } + + async waitForIdle(): Promise { + await this.agent.waitForIdle(); + } + + getTranscript(): AgentMessage[] { + return [...this.agent.state.messages]; + } } -function composeOnPayload( - first: SimpleStreamOptions["onPayload"] | undefined, - second: SimpleStreamOptions["onPayload"] | undefined, -): SimpleStreamOptions["onPayload"] | undefined { +function composeOnPayload(first: AgentOptions["onPayload"], second: AgentOptions["onPayload"]): AgentOptions["onPayload"] { if (!first) return second; if (!second) return first; - return async (payload, model) => { - const afterFirst = await first(payload, model); - return second(afterFirst ?? payload, model); + return async (payload, modelRef) => { + const afterFirst = await first(payload, modelRef); + return second(afterFirst ?? payload, modelRef); }; } diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 9fa31436..f0d2fc50 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -1,22 +1,14 @@ export * from "@earendil-works/pi-agent-core"; -export type { KernelBrowser } from "./translator/translator.js"; +export type { KernelBrowser } from "./translator/translator"; +export { createCuaComputerTools } from "./tools"; export type { - AnthropicComputerToolsOptions, + BatchDetails, ComputerToolOptions, - CuaComputerToolsOptions, - GeminiComputerToolsOptions, - OpenAIComputerToolsOptions, - TzafonComputerToolsOptions, - YutoriComputerToolsOptions, -} from "./tools.js"; -export { - createAnthropicComputerTools, - createCuaComputerTools, - createGeminiComputerTools, - createOpenAIComputerTools, - createTzafonComputerTools, - createYutoriComputerTools, -} from "./tools.js"; -export { createCuaAgent } from "./agent.js"; -export type { CreateCuaAgentOptions } from "./agent.js"; + CuaExecutorTool, + NavigationDetails, + SupportedCuaExecutorToolName, +} from "./tools"; +export { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES } from "./tools"; +export { CuaAgent, CuaHarness } from "./agent"; +export type { CuaAgentOptions, CuaHarnessOptions } from "./agent"; diff --git a/packages/agent/src/tools.ts b/packages/agent/src/tools.ts index a86c0ebb..594fad80 100644 --- a/packages/agent/src/tools.ts +++ b/packages/agent/src/tools.ts @@ -1,126 +1,94 @@ import type Kernel from "@onkernel/sdk"; import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core"; -import type { Tool } from "@earendil-works/pi-ai"; +import type { ImageContent, TextContent, Tool } from "@earendil-works/pi-ai"; import { CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, - anthropic, - gemini, - openai, - tzafon, - yutori, + CuaBatchSchema, + CuaNavigationSchema, type CuaBatchInput, - type CuaProvider, type CuaNavigationInput, } from "@onkernel/cua-ai"; -import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator.js"; -import type { ModelAction } from "./translator/types.js"; +import { InternalComputerTranslator, type KernelBrowser } from "./translator/translator"; export interface ComputerToolOptions { browser: KernelBrowser; - client?: Kernel; + client: Kernel; + toolDefinitions: Tool[]; } -export interface CuaComputerToolsOptions extends ComputerToolOptions { - provider: CuaProvider; -} +export const SUPPORTED_CUA_EXECUTOR_TOOL_NAMES = [CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME] as const; +export type SupportedCuaExecutorToolName = (typeof SUPPORTED_CUA_EXECUTOR_TOOL_NAMES)[number]; -export interface OpenAIComputerToolsOptions extends ComputerToolOptions {} -export interface AnthropicComputerToolsOptions extends ComputerToolOptions {} -export interface GeminiComputerToolsOptions extends ComputerToolOptions {} -export interface TzafonComputerToolsOptions extends ComputerToolOptions {} -export interface YutoriComputerToolsOptions extends ComputerToolOptions {} +type ToolContent = Array; -export function createCuaComputerTools(args: CuaComputerToolsOptions): AgentTool[] { - switch (args.provider) { - case "openai": - return createOpenAIComputerTools(args); - case "anthropic": - return createAnthropicComputerTools(args); - case "google": - return createGeminiComputerTools(args); - case "tzafon": - return createTzafonComputerTools(args); - case "yutori": - return createYutoriComputerTools(args); - default: - throw new Error(`unsupported CUA provider: ${String(args.provider)}`); - } -} - -export function createOpenAIComputerTools(args: OpenAIComputerToolsOptions): AgentTool[] { - return createGenericComputerTools(args, openai.createComputerToolDefinitions()); -} - -export function createAnthropicComputerTools(args: AnthropicComputerToolsOptions): AgentTool[] { - return createGenericComputerTools(args, anthropic.createComputerToolDefinitions()); -} - -export function createGeminiComputerTools(args: GeminiComputerToolsOptions): AgentTool[] { - return createGenericComputerTools(args, gemini.createComputerToolDefinitions()); -} - -export function createTzafonComputerTools(args: TzafonComputerToolsOptions): AgentTool[] { - return createGenericComputerTools(args, tzafon.createComputerToolDefinitions()); -} - -export function createYutoriComputerTools(args: YutoriComputerToolsOptions): AgentTool[] { - return createGenericComputerTools(args, yutori.createComputerToolDefinitions()); -} - -function createGenericComputerTools(args: ComputerToolOptions, definitions: Tool[]): AgentTool[] { - const translator = new InternalComputerTranslator(args); - return definitions.map((definition) => { - if (definition.name === CUA_BATCH_TOOL_NAME) { - return { - name: definition.name, - label: definition.name, - description: definition.description, - parameters: definition.parameters, - async execute(_toolCallId, params): Promise> { - const result = await executeBatchTool(translator, params as CuaBatchInput); - if (result.isError) throw Object.assign(new Error(result.details.statusText), result); - return { content: result.content, details: result.details }; - }, - }; - } - if (definition.name === CUA_NAVIGATION_TOOL_NAME) { - return { - name: definition.name, - label: definition.name, - description: definition.description, - parameters: definition.parameters, - async execute(_toolCallId, params): Promise> { - const result = await executeNavigationTool(translator, params as CuaNavigationInput); - if (result.isError) throw Object.assign(new Error(result.details.statusText), result); - return { content: result.content, details: result.details }; - }, - }; - } - throw new Error(`unsupported CUA computer tool definition: ${definition.name}`); - }); -} - -interface BatchDetails { +export interface BatchDetails { statusText: string; readResults: Array<{ type: "url"; url: string } | { type: "screenshot"; bytes: number } | { type: "cursor_position"; x: number; y: number }>; error?: string; } -interface NavigationDetails { +export interface NavigationDetails { action: string; statusText: string; url?: string; error?: string; } -async function executeBatchTool(translator: InternalComputerTranslator, params: CuaBatchInput) { - const content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = []; +type BatchTool = AgentTool; +type NavigationTool = AgentTool; +export type CuaExecutorTool = BatchTool | NavigationTool; + +export function createCuaComputerTools(args: ComputerToolOptions): CuaExecutorTool[] { + const translator = new InternalComputerTranslator(args); + return args.toolDefinitions.map((definition) => createExecutorTool(definition, translator)); +} + +function createExecutorTool(definition: Tool, translator: InternalComputerTranslator): CuaExecutorTool { + if (definition.name === CUA_BATCH_TOOL_NAME) { + const tool: BatchTool = { + name: definition.name, + label: definition.name, + description: definition.description, + parameters: CuaBatchSchema, + async execute(_toolCallId: string, params: unknown): Promise> { + const result = await executeBatchTool(translator, asBatchInput(params)); + if (result.isError) throw Object.assign(new Error(result.details.statusText), result); + return { content: result.content, details: result.details }; + }, + }; + return tool; + } + if (definition.name === CUA_NAVIGATION_TOOL_NAME) { + const tool: NavigationTool = { + name: definition.name, + label: definition.name, + description: definition.description, + parameters: CuaNavigationSchema, + async execute(_toolCallId: string, params: unknown): Promise> { + const result = await executeNavigationTool(translator, asNavigationInput(params)); + if (result.isError) throw Object.assign(new Error(result.details.statusText), result); + return { content: result.content, details: result.details }; + }, + }; + return tool; + } + throw new Error( + `unsupported CUA computer tool definition: ${definition.name}; supported names: ${SUPPORTED_CUA_EXECUTOR_TOOL_NAMES.join(", ")}`, + ); +} + +async function executeBatchTool(translator: InternalComputerTranslator, params: CuaBatchInput): Promise<{ + content: ToolContent; + details: BatchDetails; + isError: boolean; +}> { + const content: ToolContent = []; const readResults: BatchDetails["readResults"] = []; let statusText = "Actions executed successfully."; let error: Error | undefined; try { - const result = await translator.executeBatch(params.actions as unknown as ModelAction[]); + const result = await translator.executeBatch(params.actions as unknown as Array>); for (const read of result.readResults) { if (read.type === "url") { readResults.push({ type: "url", url: read.url }); @@ -146,9 +114,13 @@ async function executeBatchTool(translator: InternalComputerTranslator, params: return { content, details: { statusText, readResults, ...(error ? { error: error.message } : {}) }, isError: Boolean(error) }; } -async function executeNavigationTool(translator: InternalComputerTranslator, params: CuaNavigationInput) { +async function executeNavigationTool(translator: InternalComputerTranslator, params: CuaNavigationInput): Promise<{ + content: ToolContent; + details: NavigationDetails; + isError: boolean; +}> { const action = params.action; - const content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = []; + const content: ToolContent = []; let statusText = "Action executed successfully."; let url: string | undefined; let error: Error | undefined; @@ -169,3 +141,25 @@ async function executeNavigationTool(translator: InternalComputerTranslator, par content.unshift({ type: "text", text: statusText }); return { content, details: { action, statusText, ...(url ? { url } : {}), ...(error ? { error: error.message } : {}) }, isError: Boolean(error) }; } + +function asBatchInput(value: unknown): CuaBatchInput { + if ( + value && + typeof value === "object" && + Array.isArray((value as { actions?: unknown }).actions) + ) { + return value as CuaBatchInput; + } + throw new Error("invalid batch_computer_actions parameters"); +} + +function asNavigationInput(value: unknown): CuaNavigationInput { + if ( + value && + typeof value === "object" && + typeof (value as { action?: unknown }).action === "string" + ) { + return value as CuaNavigationInput; + } + throw new Error("invalid computer_use_extra parameters"); +} diff --git a/packages/agent/src/translator/translator.ts b/packages/agent/src/translator/translator.ts index 114add47..4e1d1bbb 100644 --- a/packages/agent/src/translator/translator.ts +++ b/packages/agent/src/translator/translator.ts @@ -1,17 +1,17 @@ import type Kernel from "@onkernel/sdk"; import type { BrowserCreateResponse, BrowserRetrieveResponse } from "@onkernel/sdk/resources/browsers"; -import type { BatchExecutionResult, ModelAction } from "./types.js"; +import type { BatchExecutionResult, ModelAction } from "./types"; export type KernelBrowser = BrowserCreateResponse | BrowserRetrieveResponse; export interface InternalComputerTranslatorOptions { browser: KernelBrowser; - client?: Kernel; + client: Kernel; } export class InternalComputerTranslator { private readonly sessionId: string; - private readonly client?: Kernel; + private readonly client: Kernel; constructor(opts: InternalComputerTranslatorOptions) { this.sessionId = opts.browser.session_id; @@ -19,8 +19,7 @@ export class InternalComputerTranslator { } async screenshotRaw(): Promise { - const client = this.requireClient(); - const response = await client.browsers.computer.captureScreenshot(this.sessionId, {}); + const response = await this.client.browsers.computer.captureScreenshot(this.sessionId, {}); return Buffer.from(await response.arrayBuffer()); } @@ -29,18 +28,18 @@ export class InternalComputerTranslator { { type: "press_key", press_key: { keys: ["Control_L", "l"] } }, { type: "press_key", press_key: { keys: ["Control_L", "c"] } }, ]); - const response = await this.requireClient().browsers.computer.readClipboard(this.sessionId); + const response = await this.client.browsers.computer.readClipboard(this.sessionId); return (response.text ?? "").trim(); } async currentMousePosition(): Promise<{ x: number; y: number }> { - const pos = await this.requireClient().browsers.computer.getMousePosition(this.sessionId); + const pos = await this.client.browsers.computer.getMousePosition(this.sessionId); return { x: toInt(pos.x), y: toInt(pos.y) }; } async executeBatch(actions: ModelAction[]): Promise { const result: BatchExecutionResult = { readResults: [] }; - const pending: any[] = []; + const pending: KernelBatchAction[] = []; const flush = async (): Promise => { if (pending.length === 0) return; @@ -89,19 +88,18 @@ export class InternalComputerTranslator { return result; } - private async runKernelBatch(actions: any[]): Promise { - await this.requireClient().browsers.computer.batch(this.sessionId, { actions }); - } - - private requireClient(): Kernel { - if (!this.client) { - throw new Error("Kernel client is required to execute computer tools for this browser"); - } - return this.client; + private async runKernelBatch(actions: KernelBatchAction[]): Promise { + await this.client.browsers.computer.batch(this.sessionId, { actions }); } } -function toSdkAction(type: string, action: ModelAction): any { +type KernelBatchAction = + Parameters[1]["actions"][number]; + +type ClickMouseButton = "back" | "forward" | "left" | "right" | "middle"; +type DragMouseButton = "left" | "right" | "middle"; + +function toSdkAction(type: string, action: ModelAction): KernelBatchAction { switch (type) { case "click": return { @@ -109,7 +107,7 @@ function toSdkAction(type: string, action: ModelAction): any { click_mouse: { x: toInt(action.x), y: toInt(action.y), - button: stringOr(action.button, "left"), + button: clickMouseButtonOr(action.button, "left"), }, }; case "double_click": @@ -128,7 +126,7 @@ function toSdkAction(type: string, action: ModelAction): any { click_mouse: { x: toInt(action.x), y: toInt(action.y), - button: stringOr(action.button, "left"), + button: clickMouseButtonOr(action.button, "left"), click_type: type === "mouse_down" ? "down" : "up", }, }; @@ -149,7 +147,7 @@ function toSdkAction(type: string, action: ModelAction): any { case "move": return { type: "move_mouse", move_mouse: { x: toInt(action.x), y: toInt(action.y) } }; case "drag": - return { type: "drag_mouse", drag_mouse: { path: toPath(action.path), button: stringOr(action.button, "left") } }; + return { type: "drag_mouse", drag_mouse: { path: toPath(action.path), button: dragMouseButtonOr(action.button, "left") } }; case "wait": return { type: "sleep", sleep: { duration_ms: typeof action.ms === "number" ? Math.trunc(action.ms) : 1000 } }; default: @@ -170,15 +168,35 @@ function stringOr(value: unknown, fallback: string): string { return typeof value === "string" && value.length > 0 ? value : fallback; } +function clickMouseButtonOr(value: unknown, fallback: ClickMouseButton): ClickMouseButton { + const candidate = stringOr(value, fallback); + if (candidate === "left" || candidate === "right" || candidate === "middle" || candidate === "back" || candidate === "forward") { + return candidate; + } + return fallback; +} + +function dragMouseButtonOr(value: unknown, fallback: DragMouseButton): DragMouseButton { + const candidate = stringOr(value, fallback); + if (candidate === "left" || candidate === "right" || candidate === "middle") { + return candidate; + } + return fallback; +} + function toStringArray(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } -function toPath(value: unknown): Array<{ x: number; y: number }> { +function toPath(value: unknown): Array<[number, number]> { if (!Array.isArray(value)) return []; - return value.map((point) => { - if (Array.isArray(point)) return { x: toInt(point[0]), y: toInt(point[1]) }; - if (point && typeof point === "object") return { x: toInt((point as Record).x), y: toInt((point as Record).y) }; - return { x: 0, y: 0 }; - }); + return value.map((point) => toPathPoint(point)); +} + +function toPathPoint(value: unknown): [number, number] { + if (Array.isArray(value)) return [toInt(value[0]), toInt(value[1])]; + if (value && typeof value === "object") { + return [toInt((value as Record).x), toInt((value as Record).y)]; + } + return [0, 0]; } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index d6fb7781..e4460810 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -1,13 +1,17 @@ import { Agent, type AgentTool } from "@earendil-works/pi-agent-core"; import { describe, expect, it } from "vitest"; -import { createCuaAgent, createCuaComputerTools, type KernelBrowser } from "../src/index.js"; +import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; +import type Kernel from "@onkernel/sdk"; +import { CuaAgent, CuaHarness, createCuaComputerTools, type KernelBrowser } from "../src/index"; const browser = { session_id: "browser_123" } as KernelBrowser; +const client = {} as Kernel; -describe("createCuaAgent", () => { - it("returns a pi-agent-core Agent and resolves model refs in initialState", () => { - const agent = createCuaAgent({ +describe("CuaAgent", () => { + it("extends pi Agent and resolves model refs in initialState", () => { + const agent = new CuaAgent({ browser, + client, initialState: { model: "openai:gpt-5.5", }, @@ -19,7 +23,7 @@ describe("createCuaAgent", () => { }); it("uses provided tools exactly", () => { - const tool: AgentTool = { + const tool: AgentTool = { name: "custom", label: "custom", description: "custom tool", @@ -29,8 +33,9 @@ describe("createCuaAgent", () => { }, }; - const agent = createCuaAgent({ + const agent = new CuaAgent({ browser, + client, initialState: { model: "yutori:n1.5-latest", tools: [tool], @@ -41,8 +46,9 @@ describe("createCuaAgent", () => { }); it("lets users explicitly compose default tools", () => { + const runtime = resolveCuaRuntimeSpec("openai:gpt-5.5"); const tools = [ - ...createCuaComputerTools({ provider: "openai", browser }), + ...createCuaComputerTools({ browser, client, toolDefinitions: runtime.toolDefinitions }), { name: "custom", label: "custom", @@ -54,8 +60,9 @@ describe("createCuaAgent", () => { } satisfies AgentTool, ]; - const agent = createCuaAgent({ + const agent = new CuaAgent({ browser, + client, initialState: { model: "openai:gpt-5.5", tools, @@ -67,3 +74,28 @@ describe("createCuaAgent", () => { expect(agent.state.systemPrompt).toBe("Use the browser carefully."); }); }); + +describe("CuaHarness", () => { + it("wraps a pi Agent and resolves model refs", () => { + const harness = new CuaHarness({ + browser, + client, + model: "openai:gpt-5.5", + getApiKey: () => "test-key", + }); + expect(harness.agent).toBeInstanceOf(Agent); + expect(harness.agent.state.model.id).toBe("gpt-5.5"); + expect(harness.agent.state.tools.length).toBeGreaterThan(0); + }); + + it("exposes transcript snapshots", () => { + const harness = new CuaHarness({ + browser, + client, + model: "openai:gpt-5.5", + }); + const transcript = harness.getTranscript(); + expect(transcript).toEqual(harness.state.messages); + expect(transcript).not.toBe(harness.state.messages); + }); +}); diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts new file mode 100644 index 00000000..cba00632 --- /dev/null +++ b/packages/agent/test/e2e.live.test.ts @@ -0,0 +1,223 @@ +import Kernel from "@onkernel/sdk"; +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import { describe, expect, it } from "vitest"; +import { CuaAgent, CuaHarness } from "../src/index"; + +const LIVE = process.env.CUA_E2E_LIVE === "1"; +const KERNEL_API_KEY = process.env.KERNEL_API_KEY; + +type ProviderCase = { + name: string; + apiKeyEnvVar: string; + modelRef: + | "openai:gpt-5.5" + | "anthropic:claude-opus-4-7" + | "google:gemini-3-flash-preview" + | "tzafon:tzafon.northstar-cua-fast" + | "yutori:n1.5-latest"; + prompt: string; + expectToolCalls: boolean; + timeoutMs: number; +}; + +const cases: ProviderCase[] = [ + { + name: "openai", + apiKeyEnvVar: "OPENAI_API_KEY", + modelRef: "openai:gpt-5.5", + prompt: [ + "Call batch_computer_actions exactly once.", + 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Do not call any other tools.", + "Then provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: true, + timeoutMs: 120_000, + }, + { + name: "anthropic", + apiKeyEnvVar: "ANTHROPIC_API_KEY", + modelRef: "anthropic:claude-opus-4-7", + prompt: [ + "Call batch_computer_actions exactly once.", + 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Do not call any other tools.", + "Then provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: true, + timeoutMs: 120_000, + }, + { + name: "gemini", + apiKeyEnvVar: "GOOGLE_API_KEY", + modelRef: "google:gemini-3-flash-preview", + prompt: [ + "Call batch_computer_actions exactly once.", + 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Do not call any other tools.", + "Then provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: true, + timeoutMs: 300_000, + }, + { + name: "tzafon", + apiKeyEnvVar: "TZAFON_API_KEY", + modelRef: "tzafon:tzafon.northstar-cua-fast", + prompt: [ + "Call batch_computer_actions exactly once.", + 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Do not call any other tools.", + "Then provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: false, + timeoutMs: 120_000, + }, + { + name: "yutori", + apiKeyEnvVar: "YUTORI_API_KEY", + modelRef: "yutori:n1.5-latest", + prompt: [ + "Call batch_computer_actions exactly once.", + 'Pass this exact arguments JSON: {"actions":[{"type":"screenshot"}]}', + "Do not call any other tools.", + "Then provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: false, + timeoutMs: 180_000, + }, +]; + +type RunStats = { + toolCalls: number; + toolResults: number; + hasReadArtifact: boolean; + finalAssistant?: AgentMessage; + toolErrors: string[]; + assistantErrors: string[]; +}; + +function shouldRunCase(c: ProviderCase): boolean { + if (!LIVE) return false; + if (!KERNEL_API_KEY) return false; + return Boolean(process.env[c.apiKeyEnvVar]); +} + +async function withBrowser(run: (client: Kernel, browser: Awaited>) => Promise): Promise { + if (!KERNEL_API_KEY) { + throw new Error("KERNEL_API_KEY is required"); + } + const client = new Kernel({ apiKey: KERNEL_API_KEY }); + const browser = await client.browsers.create({ stealth: true }); + try { + return await run(client, browser); + } finally { + await client.browsers.deleteByID(browser.session_id).catch(() => {}); + } +} + +function assertStats(stats: RunStats, expectToolCalls: boolean, providerName: string, runtimeName: "agent" | "harness"): void { + if (expectToolCalls) { + expect(stats.toolCalls).toBeGreaterThan(0); + expect(stats.toolResults).toBeGreaterThan(0); + expect(stats.hasReadArtifact).toBe(true); + } + expect(stats.toolErrors, `${providerName}/${runtimeName} emitted tool errors: ${stats.toolErrors.join(" | ")}`).toHaveLength(0); + expect(stats.assistantErrors, `${providerName}/${runtimeName} emitted assistant errors: ${stats.assistantErrors.join(" | ")}`).toHaveLength(0); + expect(stats.finalAssistant).toBeDefined(); + if (stats.finalAssistant?.role === "assistant") { + expect(stats.finalAssistant.stopReason, `${providerName}/${runtimeName} ended in assistant error`).not.toBe("error"); + expect(stats.finalAssistant.stopReason, `${providerName}/${runtimeName} ended in assistant abort`).not.toBe("aborted"); + } +} + +describe("Cua live e2e", () => { + for (const c of cases) { + const test = shouldRunCase(c) ? it : it.skip; + + test( + `${c.name}: CuaAgent executes browser steps`, + async () => { + await withBrowser(async (client, browser) => { + const stats: RunStats = { toolCalls: 0, toolResults: 0, hasReadArtifact: false, toolErrors: [], assistantErrors: [] }; + const agent = new CuaAgent({ + browser, + client, + getApiKey: () => process.env[c.apiKeyEnvVar], + initialState: { + model: c.modelRef, + }, + }); + agent.subscribe((event) => { + if (event.type === "tool_execution_start") stats.toolCalls += 1; + if (event.type === "tool_execution_end" && event.isError) { + stats.toolErrors.push(`${event.toolName}: failed`); + } + if (event.type === "message_end" && event.message.role === "toolResult") { + stats.toolResults += 1; + if ( + event.message.content.some( + (block) => block.type === "image" || (block.type === "text" && /url\(\)|Current URL:/.test(block.text)), + ) + ) { + stats.hasReadArtifact = true; + } + } + if (event.type === "message_end" && event.message.role === "assistant") { + stats.finalAssistant = event.message; + if (event.message.errorMessage) { + stats.assistantErrors.push(event.message.errorMessage); + } + } + }); + + await agent.prompt(c.prompt); + assertStats(stats, c.expectToolCalls, c.name, "agent"); + }); + }, + c.timeoutMs, + ); + + test( + `${c.name}: CuaHarness executes browser steps`, + async () => { + await withBrowser(async (client, browser) => { + const stats: RunStats = { toolCalls: 0, toolResults: 0, hasReadArtifact: false, toolErrors: [], assistantErrors: [] }; + const harness = new CuaHarness({ + browser, + client, + model: c.modelRef, + getApiKey: () => process.env[c.apiKeyEnvVar], + }); + + harness.subscribe((event) => { + if (event.type === "tool_execution_start") stats.toolCalls += 1; + if (event.type === "tool_execution_end" && event.isError) { + stats.toolErrors.push(`${event.toolName}: failed`); + } + if (event.type === "message_end" && event.message.role === "toolResult") { + stats.toolResults += 1; + if ( + event.message.content.some( + (block) => block.type === "image" || (block.type === "text" && /url\(\)|Current URL:/.test(block.text)), + ) + ) { + stats.hasReadArtifact = true; + } + } + if (event.type === "message_end" && event.message.role === "assistant") { + stats.finalAssistant = event.message; + if (event.message.errorMessage) { + stats.assistantErrors.push(event.message.errorMessage); + } + } + }); + + await harness.prompt(c.prompt); + assertStats(stats, c.expectToolCalls, c.name, "harness"); + }); + }, + c.timeoutMs, + ); + } +}); diff --git a/packages/agent/test/tool-exhaustiveness.test.ts b/packages/agent/test/tool-exhaustiveness.test.ts new file mode 100644 index 00000000..1cb4877b --- /dev/null +++ b/packages/agent/test/tool-exhaustiveness.test.ts @@ -0,0 +1,45 @@ +import { Type, type Tool } from "@onkernel/cua-ai"; +import { CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; +import type Kernel from "@onkernel/sdk"; +import { describe, expect, it } from "vitest"; +import { SUPPORTED_CUA_EXECUTOR_TOOL_NAMES, createCuaComputerTools, type KernelBrowser } from "../src/index"; + +const browser = { session_id: "browser_123" } as KernelBrowser; +const client = {} as Kernel; + +describe("Cua tool executor coverage", () => { + it("covers every canonical tool name exported by cua-ai defaults", () => { + const names = new Set(); + for (const provider of CUA_PROVIDERS) { + const model = listCuaModels(provider)[0]; + expect(model, `no CUA model for provider ${provider}`).toBeDefined(); + for (const definition of resolveCuaRuntimeSpec(model!.ref).toolDefinitions) { + names.add(definition.name); + } + } + expect([...names].sort()).toEqual([...SUPPORTED_CUA_EXECUTOR_TOOL_NAMES].sort()); + }); + + it("instantiates one executor per canonical definition", () => { + const toolDefinitions = resolveCuaRuntimeSpec("openai:gpt-5.5").toolDefinitions; + const tools = createCuaComputerTools({ browser, client, toolDefinitions }); + expect(tools.map((tool) => tool.name).sort()).toEqual(toolDefinitions.map((tool) => tool.name).sort()); + }); + + it("fails fast on unsupported tool names", () => { + const unsupportedDefinitions: Tool[] = [ + { + name: "unknown_tool", + description: "unsupported", + parameters: Type.Object({}), + }, + ]; + expect(() => + createCuaComputerTools({ + browser, + client, + toolDefinitions: unsupportedDefinitions, + }), + ).toThrow(/unsupported CUA computer tool definition/); + }); +}); diff --git a/packages/agent/vitest.config.ts b/packages/agent/vitest.config.ts index 361d790c..039b5887 100644 --- a/packages/agent/vitest.config.ts +++ b/packages/agent/vitest.config.ts @@ -1,6 +1,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ + server: { + host: "127.0.0.1", + }, test: { globals: true, environment: "node", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index e7b25bd6..4c2a3cdd 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,6 @@ ## 0.1.0 -- Initial `@onkernel/cua-ai` package. -- Added provider-qualified CUA model refs. -- Added local CUA provider registration and Tzafon/Yutori pi-ai adapters. +- Provider-qualified CUA model catalog with support annotations and curated overrides. +- Unified runtime-spec resolution for provider defaults (tools, prompts, payload middleware). +- Registers CUA provider adapters and exports canonical computer-use schemas/tool definitions. diff --git a/packages/ai/README.md b/packages/ai/README.md index 16084ea3..1ac75c11 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -102,7 +102,18 @@ Top-level exports: - `getCuaModel(ref: CuaModelRef): Model` - `listCuaModels(provider?: CuaProvider): CuaModelInfo[]` - `providerForModel(model: Model): CuaProvider` -- `isCuaProvider(value: string): value is CuaProvider` +- `resolveCuaRuntimeSpec(input: CuaModelRef | Model): CuaRuntimeSpec` +- `CUA_PROVIDERS: readonly CuaProvider[]` +- `CuaBatchSchema`, `CuaActionSchema`, `CuaNavigationSchema` TypeBox schemas +- `createCuaActionSchema(actions?)`, `createCuaBatchSchema(actions?)` + +`resolveCuaRuntimeSpec()` centralizes provider-specific defaults for +runtime consumers: + +- canonical provider id +- canonical CUA tool definitions +- default system prompt text +- optional provider payload middleware (for protocol quirks) Provider namespaces expose `createComputerToolDefinitions({ actions? })` for building model-facing pi-ai `Tool[]` definitions. Omit `actions` for the diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index 17778f35..e1b34fd1 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -1,7 +1,7 @@ import { readFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { complete, getCuaModel, openai } from "../src/index.js"; +import { complete, getCuaModel, openai } from "../src/index"; const here = dirname(fileURLToPath(import.meta.url)); const screenshotPath = join(here, "screenshot.png"); diff --git a/packages/ai/package.json b/packages/ai/package.json index 4787cc2d..dc1ed2a4 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -8,6 +8,7 @@ "exports": { ".": { "types": "./dist/index.d.ts", + "source": "./src/index.ts", "import": "./dist/index.js" } }, diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts new file mode 100644 index 00000000..edf04be3 --- /dev/null +++ b/packages/ai/src/api-keys.ts @@ -0,0 +1,52 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import { parseCuaModelRef, providerForModel, type CuaModelRef, type CuaProvider } from "./models"; + +/** + * Environment variables accepted for each CUA provider. + * + * This mirrors pi-ai's approach: model lookup is pure, while auth is resolved + * when streaming. These helpers let callers share one readable convention for + * explicit `getApiKey` wiring (especially useful for `google` vs `gemini`). + */ +const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { + openai: ["OPENAI_API_KEY"], + anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], + google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], + tzafon: ["TZAFON_API_KEY"], + yutori: ["YUTORI_API_KEY"], +}; + +export function cuaApiKeyEnvVarsForProvider(provider: string): readonly string[] { + if (provider === "gemini") { + return CUA_PROVIDER_API_KEY_ENV_VARS.google; + } + return CUA_PROVIDER_API_KEY_ENV_VARS[provider as keyof typeof CUA_PROVIDER_API_KEY_ENV_VARS] ?? []; +} + +export function getCuaEnvApiKey(provider: string): string | undefined { + for (const envVar of cuaApiKeyEnvVarsForProvider(provider)) { + const value = process.env[envVar]; + if (value?.trim()) return value; + } + return undefined; +} + +export function requireCuaEnvApiKey(provider: string): string { + const apiKey = getCuaEnvApiKey(provider); + if (apiKey) return apiKey; + const envVars = cuaApiKeyEnvVarsForProvider(provider); + if (envVars.length === 0) { + throw new Error(`No known API key environment variables for provider "${provider}"`); + } + throw new Error(`Missing API key for "${provider}". Set one of: ${envVars.join(", ")}`); +} + +export function getCuaEnvApiKeyForModel(input: CuaModelRef | Model): string | undefined { + const provider = typeof input === "string" ? parseCuaModelRef(input).provider : providerForModel(input); + return getCuaEnvApiKey(provider); +} + +export function requireCuaEnvApiKeyForModel(input: CuaModelRef | Model): string { + const provider = typeof input === "string" ? parseCuaModelRef(input).provider : providerForModel(input); + return requireCuaEnvApiKey(provider); +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index b00c52f9..11ce4b5f 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,50 +1,15 @@ -import { registerCuaProviders } from "./providers.js"; +import { registerCuaProviders } from "./providers"; export * from "@earendil-works/pi-ai"; -export { - getCuaModel, - isCuaProvider, - listCuaModels, - providerForModel, -} from "./models.js"; -export type { CuaModelInfo, CuaModelRef, CuaProvider } from "./models.js"; - -export { - CUA_ACTION_TYPES, - CUA_BATCH_TOOL_NAME, - CUA_NAVIGATION_TOOL_NAME, - createComputerToolDefinitions, -} from "./providers/common.js"; -export type { - ComputerToolCoordinateSystem, - CreateComputerToolDefinitionsOptions, - CuaAction, - CuaActionBack, - CuaActionClick, - CuaActionCursorPosition, - CuaActionDoubleClick, - CuaActionDrag, - CuaActionForward, - CuaActionGoto, - CuaActionKeypress, - CuaActionMouseDown, - CuaActionMouseUp, - CuaActionMove, - CuaActionScreenshot, - CuaActionScroll, - CuaActionType, - CuaActionTypeText, - CuaActionUrl, - CuaActionWait, - CuaBatchInput, - CuaNavigationInput, -} from "./providers/common.js"; - -export * as anthropic from "./providers/anthropic/index.js"; -export * as gemini from "./providers/gemini/index.js"; -export * as openai from "./providers/openai/index.js"; -export * as tzafon from "./providers/tzafon/index.js"; -export * as yutori from "./providers/yutori/index.js"; +export * from "./models"; +export * from "./api-keys"; +export * from "./runtime-spec"; +export * from "./providers/common"; +export * as anthropic from "./providers/anthropic/index"; +export * as gemini from "./providers/gemini/index"; +export * as openai from "./providers/openai/index"; +export * as tzafon from "./providers/tzafon/index"; +export * as yutori from "./providers/yutori/index"; registerCuaProviders(); diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 69cacb9d..9a7962f1 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -1,6 +1,6 @@ import { registerApiProvider } from "@earendil-works/pi-ai"; -import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider.js"; -import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider.js"; +import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; +import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; let registered = false; diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 466d35d2..7252c349 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem } from "../common.js"; +import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as ANTHROPIC_CUA_ACTION_TYPES, @@ -8,12 +8,12 @@ export { createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as AnthropicBatchSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as AnthropicAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as AnthropicBatchInput, -} from "../common.js"; +} from "../common"; // Provider-native action vocabulary emitted on `tool_use.input.action`. Latest // tool version is `computer_20251124`, which extends earlier dated versions: diff --git a/packages/ai/src/providers/gemini/index.ts b/packages/ai/src/providers/gemini/index.ts index 5e12ae6a..74b6b6e0 100644 --- a/packages/ai/src/providers/gemini/index.ts +++ b/packages/ai/src/providers/gemini/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem } from "../common.js"; +import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as GEMINI_CUA_ACTION_TYPES, @@ -8,12 +8,12 @@ export { createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as GeminiBatchSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as GeminiAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as GeminiBatchInput, -} from "../common.js"; +} from "../common"; // Provider-native function names emitted on `functionCall.name` (PREDEFINED_COMPUTER_USE_FUNCTIONS): // open_web_browser, click_at, hover_at, type_text_at, scroll_document, diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index 609c027c..6850ab23 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem } from "../common.js"; +import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as OPENAI_CUA_ACTION_TYPES, @@ -11,13 +11,13 @@ export { createCuaBatchSchema as createBatchSchema, CuaBatchSchema as OpenAIBatchSchema, CuaNavigationSchema as OpenAIExtraSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as OpenAIAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as OpenAIBatchInput, CuaNavigationInput as OpenAIExtraInput, -} from "../common.js"; +} from "../common"; // Provider-native action vocabulary emitted on `computer_call.action.type`: // click, double_click, drag, move, scroll, type, keypress, wait, screenshot @@ -29,3 +29,13 @@ export const OPENAI_BATCH_INSTRUCTIONS = `You have two browser tools: 2. computer_use_extra for a single high-level goto, back, forward, or url action. Prefer batch_computer_actions for predictable multi-step browser interaction. Include explicit url(), cursor_position(), or screenshot() read steps when you need intermediate state.`; + +export function openaiResponsesStoreOnPayload(payload: unknown): unknown | undefined { + if (!payload || typeof payload !== "object") return undefined; + const current = payload as Record; + if (current.store === true) return undefined; + return { + ...current, + store: true, + }; +} diff --git a/packages/ai/src/providers/tzafon/index.ts b/packages/ai/src/providers/tzafon/index.ts index be8cb191..1df4a648 100644 --- a/packages/ai/src/providers/tzafon/index.ts +++ b/packages/ai/src/providers/tzafon/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem } from "../common.js"; +import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as TZAFON_ACTION_TYPES, @@ -8,17 +8,17 @@ export { createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as TzafonBatchSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as TzafonAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as TzafonBatchInput, -} from "../common.js"; +} from "../common"; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses, -} from "./provider.js"; +} from "./provider"; // Provider-native action vocabulary. The model card lists supported actions; // the Responses API loop dispatches on `action.type` and adds terminal control diff --git a/packages/ai/src/providers/yutori/index.ts b/packages/ai/src/providers/yutori/index.ts index 20efc201..7904b9f7 100644 --- a/packages/ai/src/providers/yutori/index.ts +++ b/packages/ai/src/providers/yutori/index.ts @@ -1,4 +1,4 @@ -import type { ComputerToolCoordinateSystem } from "../common.js"; +import type { ComputerToolCoordinateSystem } from "../common"; export { CUA_ACTION_TYPES as YUTORI_ACTION_TYPES, @@ -8,18 +8,18 @@ export { createCuaActionSchema as createActionSchema, createCuaBatchSchema as createBatchSchema, CuaBatchSchema as YutoriBatchSchema, -} from "../common.js"; +} from "../common"; export type { CuaAction as YutoriAction, CreateComputerToolDefinitionsOptions, CuaBatchInput as YutoriBatchInput, -} from "../common.js"; +} from "../common"; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori, yutoriBuiltinToolsOnPayload, -} from "./provider.js"; +} from "./provider"; // Provider-native action vocabulary differs between Navigator versions: // n1 (fixed tool set): diff --git a/packages/ai/src/providers/yutori/provider.ts b/packages/ai/src/providers/yutori/provider.ts index 2588d56e..1fd60b86 100644 --- a/packages/ai/src/providers/yutori/provider.ts +++ b/packages/ai/src/providers/yutori/provider.ts @@ -13,7 +13,7 @@ import { type TextContent, type ToolCall, } from "@earendil-works/pi-ai"; -import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, type CuaAction } from "../common.js"; +import { CUA_ACTION_TYPES, CUA_BATCH_TOOL_NAME, type CuaAction } from "../common"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts new file mode 100644 index 00000000..6ef6811e --- /dev/null +++ b/packages/ai/src/runtime-spec.ts @@ -0,0 +1,79 @@ +import type { Api, Model, SimpleStreamOptions, Tool } from "@earendil-works/pi-ai"; +import { type CuaModelRef, type CuaProvider, getCuaModel, providerForModel } from "./models"; +import * as anthropic from "./providers/anthropic/index"; +import * as gemini from "./providers/gemini/index"; +import * as openai from "./providers/openai/index"; +import * as tzafon from "./providers/tzafon/index"; +import * as yutori from "./providers/yutori/index"; + +/** + * Provider-resolved runtime defaults for CUA execution. + * + * `@onkernel/cua-agent` consumes this shape so it can stay provider-neutral: + * it only needs model, tool definitions, and optional payload middleware, + * without branching on provider-specific quirks. + */ +export interface CuaRuntimeSpec { + model: Model; + provider: CuaProvider; + /** Canonical CUA tool definitions exposed to the model. */ + toolDefinitions: Tool[]; + /** Provider-tuned baseline prompt for browser control behavior. */ + defaultSystemPrompt: string; + /** Optional provider middleware for request payload adaptation. */ + onPayload?: SimpleStreamOptions["onPayload"]; +} + +export type CuaRuntimeSpecInput = CuaModelRef | Model; + +/** + * Resolve provider-owned policy from either a CUA model ref or a concrete model. + * + * This is intentionally the only place where provider-specific defaults are + * selected. Callers should consume the returned spec and avoid branching on + * provider names directly. + */ +export function resolveCuaRuntimeSpec(input: CuaRuntimeSpecInput): CuaRuntimeSpec { + const model = typeof input === "string" ? getCuaModel(input) : input; + const provider = providerForModel(model); + switch (provider) { + case "anthropic": + return { + model, + provider, + toolDefinitions: anthropic.createComputerToolDefinitions(), + defaultSystemPrompt: anthropic.buildAnthropicSystemPrompt(), + }; + case "google": + return { + model, + provider, + toolDefinitions: gemini.createComputerToolDefinitions(), + defaultSystemPrompt: gemini.buildGeminiSystemPrompt(), + }; + case "tzafon": + return { + model, + provider, + toolDefinitions: tzafon.createComputerToolDefinitions(), + defaultSystemPrompt: tzafon.buildTzafonSystemPrompt(), + }; + case "yutori": + return { + model, + provider, + toolDefinitions: yutori.createComputerToolDefinitions(), + defaultSystemPrompt: yutori.buildYutoriSystemPrompt(), + onPayload: yutori.yutoriBuiltinToolsOnPayload, + }; + case "openai": + default: + return { + model, + provider, + toolDefinitions: openai.createComputerToolDefinitions(), + defaultSystemPrompt: openai.OPENAI_BATCH_INSTRUCTIONS, + onPayload: openai.openaiResponsesStoreOnPayload, + }; + } +} diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts new file mode 100644 index 00000000..501b0106 --- /dev/null +++ b/packages/ai/test/api-keys.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + cuaApiKeyEnvVarsForProvider, + getCuaEnvApiKey, + getCuaEnvApiKeyForModel, + requireCuaEnvApiKey, +} from "../src/index"; + +const ENV_KEYS = [ + "OPENAI_API_KEY", + "ANTHROPIC_OAUTH_TOKEN", + "ANTHROPIC_API_KEY", + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "TZAFON_API_KEY", + "YUTORI_API_KEY", +] as const; + +const ORIGINAL_ENV = new Map(ENV_KEYS.map((key) => [key, process.env[key]])); + +afterEach(() => { + for (const key of ENV_KEYS) { + const value = ORIGINAL_ENV.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +describe("cua api key helpers", () => { + it("maps provider names to expected environment variables", () => { + expect(cuaApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("unknown")).toEqual([]); + }); + + it("resolves provider api keys with fallback order", () => { + process.env.GEMINI_API_KEY = "gemini"; + expect(getCuaEnvApiKey("google")).toBe("gemini"); + process.env.GOOGLE_API_KEY = "google"; + expect(getCuaEnvApiKey("google")).toBe("google"); + }); + + it("resolves keys from model refs", () => { + process.env.OPENAI_API_KEY = "openai"; + expect(getCuaEnvApiKeyForModel("openai:gpt-5.5")).toBe("openai"); + }); + + it("throws readable errors when missing", () => { + delete process.env.TZAFON_API_KEY; + expect(() => requireCuaEnvApiKey("tzafon")).toThrow("TZAFON_API_KEY"); + }); +}); diff --git a/packages/ai/test/batch-tool.integration.test.ts b/packages/ai/test/batch-tool.integration.test.ts index c1cc540a..5ee0b8b3 100644 --- a/packages/ai/test/batch-tool.integration.test.ts +++ b/packages/ai/test/batch-tool.integration.test.ts @@ -15,7 +15,7 @@ import { openai, tzafon, yutori, -} from "../src/index.js"; +} from "../src/index"; const here = dirname(fileURLToPath(import.meta.url)); const screenshotPath = join(here, "..", "examples", "screenshot.png"); @@ -258,7 +258,7 @@ describe("batch_computer_actions multi-action sequences", () => { } expect(batchCalls.length).toBe(1); const args = batchCalls[0]!.arguments as { actions: Array> }; - expect(args.actions.length).toBe(1); + expect(args.actions.length).toBeGreaterThanOrEqual(1); }, 60_000); } } diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index 836991fb..1b9fd07e 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -6,7 +6,7 @@ import { findCuaAnnotation, formatCuaModelRef, parseCuaModelRef, -} from "../src/models.js"; +} from "../src/index"; describe("CUA model refs", () => { it("parses and formats provider-qualified refs", () => { diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index c39b322c..3903ba16 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getApiProvider, tzafon, yutori } from "../src/index.js"; +import { getApiProvider, tzafon, yutori } from "../src/index"; const TZAFON_RESPONSES_API = tzafon.TZAFON_RESPONSES_API; const YUTORI_CHAT_COMPLETIONS_API = yutori.YUTORI_CHAT_COMPLETIONS_API; diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts new file mode 100644 index 00000000..dc6df8e7 --- /dev/null +++ b/packages/ai/test/runtime-spec.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { CUA_BATCH_TOOL_NAME, CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, listCuaModels, resolveCuaRuntimeSpec } from "../src/index"; + +describe("resolveCuaRuntimeSpec", () => { + it("resolves a runtime spec for every CUA provider", () => { + for (const provider of CUA_PROVIDERS) { + const model = listCuaModels(provider)[0]; + expect(model, `no CUA model configured for provider ${provider}`).toBeDefined(); + + const spec = resolveCuaRuntimeSpec(model!.ref); + expect(spec.provider).toBe(provider); + expect(spec.model.id).toBe(model!.model); + expect(spec.toolDefinitions.length).toBeGreaterThan(0); + expect(typeof spec.defaultSystemPrompt).toBe("string"); + expect(spec.defaultSystemPrompt.length).toBeGreaterThan(0); + expect(spec.toolDefinitions.map((tool) => tool.name)).toContain(CUA_BATCH_TOOL_NAME); + expect(spec.toolDefinitions.map((tool) => tool.name)).toContain(CUA_NAVIGATION_TOOL_NAME); + } + }); + + it("only sets payload middleware for providers that need it", () => { + const yutoriSpec = resolveCuaRuntimeSpec("yutori:n1.5-latest"); + const openaiSpec = resolveCuaRuntimeSpec("openai:gpt-5.5"); + const anthropicSpec = resolveCuaRuntimeSpec("anthropic:claude-opus-4-7"); + expect(yutoriSpec.onPayload).toBeTypeOf("function"); + expect(openaiSpec.onPayload).toBeTypeOf("function"); + expect(anthropicSpec.onPayload).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/tools.test.ts b/packages/ai/test/tools.test.ts index 03de9a12..8b15c469 100644 --- a/packages/ai/test/tools.test.ts +++ b/packages/ai/test/tools.test.ts @@ -9,7 +9,7 @@ import { openai, tzafon, yutori, -} from "../src/index.js"; +} from "../src/index"; const providers = { openai, anthropic, gemini, tzafon, yutori }; diff --git a/packages/ai/test/yutori-payload.test.ts b/packages/ai/test/yutori-payload.test.ts new file mode 100644 index 00000000..dc27f85f --- /dev/null +++ b/packages/ai/test/yutori-payload.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { yutori } from "../src/index"; + +describe("yutoriBuiltinToolsOnPayload", () => { + it("removes yutori built-in tool definitions from payload.tools", () => { + const payload = { + tools: [ + { type: "function", function: { name: "click" } }, + { type: "function", function: { name: "move" } }, + { type: "function", function: { name: "batch_computer_actions" } }, + ], + }; + const next = yutori.yutoriBuiltinToolsOnPayload(payload) as { tools?: Array<{ function?: { name?: string } }> }; + expect(next.tools).toEqual([{ type: "function", function: { name: "batch_computer_actions" } }]); + }); + + it("returns undefined for non-object payloads", () => { + expect(yutori.yutoriBuiltinToolsOnPayload(undefined)).toBeUndefined(); + expect(yutori.yutoriBuiltinToolsOnPayload("x")).toBeUndefined(); + }); +}); diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 361d790c..039b5887 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -1,6 +1,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ + server: { + host: "127.0.0.1", + }, test: { globals: true, environment: "node", diff --git a/packages/cua-anthropic/src/batch-tool.ts b/packages/cua-anthropic/src/batch-tool.ts index 53e2826c..d8dbd40c 100644 --- a/packages/cua-anthropic/src/batch-tool.ts +++ b/packages/cua-anthropic/src/batch-tool.ts @@ -7,7 +7,7 @@ import { executeAnthropicBatch, ANTHROPIC_BATCH_DESCRIPTION, ANTHROPIC_BATCH_TOOL_NAME, -} from "./batch.js"; +} from "./batch"; /** * Anthropic-flavored `batch_computer_actions` tool. diff --git a/packages/cua-anthropic/src/batch.ts b/packages/cua-anthropic/src/batch.ts index 35189a20..be2572a0 100644 --- a/packages/cua-anthropic/src/batch.ts +++ b/packages/cua-anthropic/src/batch.ts @@ -1,7 +1,7 @@ import type { ComputerTranslator, ComputerUseToolResult, ModelAction } from "@onkernel/cua-translator"; import { type Static, Type } from "@sinclair/typebox"; -import { ANTHROPIC_CUA_EXTRA_ACTION_TYPES } from "./cua-extras.js"; -import { ANTHROPIC_OFFICIAL_ACTION_TYPES } from "./official.js"; +import { ANTHROPIC_CUA_EXTRA_ACTION_TYPES } from "./cua-extras"; +import { ANTHROPIC_OFFICIAL_ACTION_TYPES } from "./official"; const COORD_PAIR = Type.Object({ x: Type.Number(), y: Type.Number() }, { additionalProperties: false }); diff --git a/packages/cua-anthropic/src/computer-tool.ts b/packages/cua-anthropic/src/computer-tool.ts index 41dc36c7..d3cb0d48 100644 --- a/packages/cua-anthropic/src/computer-tool.ts +++ b/packages/cua-anthropic/src/computer-tool.ts @@ -5,7 +5,7 @@ import { ComputerSchema, type AnthropicComputerDetails, executeAnthropicComputerAction, -} from "./computer.js"; +} from "./computer"; /** * Build the `computer` AgentTool for Anthropic. The provider payload hook diff --git a/packages/cua-anthropic/src/index.ts b/packages/cua-anthropic/src/index.ts index 4c48ac34..179072a5 100644 --- a/packages/cua-anthropic/src/index.ts +++ b/packages/cua-anthropic/src/index.ts @@ -20,17 +20,17 @@ export { ANTHROPIC_BATCH_TOOL_WIRE_SPEC, executeAnthropicBatch, expandTripleClick, -} from "./batch.js"; +} from "./batch"; export type { AnthropicBatchToolDetails, AnthropicBatchToolInput, -} from "./batch.js"; +} from "./batch"; export { ComputerSchema, executeAnthropicComputerAction, translateAnthropicAction, -} from "./computer.js"; -export type { AnthropicComputerInput, AnthropicComputerDetails } from "./computer.js"; +} from "./computer"; +export type { AnthropicComputerInput, AnthropicComputerDetails } from "./computer"; export { ANTHROPIC_COMPUTER_TOOL, ANTHROPIC_COMPACTION_BETA, @@ -40,24 +40,24 @@ export { ANTHROPIC_COMPUTER_USE_BETA, ANTHROPIC_OFFICIAL_ACTION_TYPES, anthropicSupportsCompaction, -} from "./official.js"; +} from "./official"; export type { AnthropicComputerToolVersion, AnthropicOfficialActionType, -} from "./official.js"; -export { ANTHROPIC_CUA_EXTRA_ACTION_TYPES } from "./cua-extras.js"; -export type { AnthropicCuaExtraActionType } from "./cua-extras.js"; -export { anthropic } from "./model.js"; +} from "./official"; +export { ANTHROPIC_CUA_EXTRA_ACTION_TYPES } from "./cua-extras"; +export type { AnthropicCuaExtraActionType } from "./cua-extras"; +export { anthropic } from "./model"; export type { AnthropicModelOptions, AnthropicModelRunDetails, -} from "./model.js"; +} from "./model"; export { compactAnthropicMessagesForRequest, hasAnthropicCompactionBlock, -} from "./context.js"; +} from "./context"; export { buildAnthropicSystemPrompt, ANTHROPIC_INSTRUCTIONS_RAW, -} from "./system-prompt.js"; -export type { AnthropicSystemPromptOptions } from "./system-prompt.js"; +} from "./system-prompt"; +export type { AnthropicSystemPromptOptions } from "./system-prompt"; diff --git a/packages/cua-anthropic/src/model.ts b/packages/cua-anthropic/src/model.ts index eeb9cfa0..4cdc2374 100644 --- a/packages/cua-anthropic/src/model.ts +++ b/packages/cua-anthropic/src/model.ts @@ -5,11 +5,11 @@ import type { ComputerUseToolResult, } from "@onkernel/cua-translator"; import { Anthropic } from "@anthropic-ai/sdk"; -import { ANTHROPIC_BATCH_TOOL_WIRE_SPEC, executeAnthropicBatch } from "./batch.js"; +import { ANTHROPIC_BATCH_TOOL_WIRE_SPEC, executeAnthropicBatch } from "./batch"; import { type AnthropicComputerInput, executeAnthropicComputerAction, -} from "./computer.js"; +} from "./computer"; import { anthropicComputerToolForModel, anthropicComputerUseBetaForModel, @@ -17,9 +17,9 @@ import { ANTHROPIC_COMPACTION_EDIT_TYPE, ANTHROPIC_COMPACTION_MIN_TRIGGER_TOKENS, anthropicSupportsCompaction, -} from "./official.js"; -import { compactAnthropicMessagesForRequest } from "./context.js"; -import { buildAnthropicSystemPrompt } from "./system-prompt.js"; +} from "./official"; +import { compactAnthropicMessagesForRequest } from "./context"; +import { buildAnthropicSystemPrompt } from "./system-prompt"; export interface AnthropicModelOptions { apiKey?: string; diff --git a/packages/cua-anthropic/src/payload-hook.ts b/packages/cua-anthropic/src/payload-hook.ts index b93dba83..69c67b3d 100644 --- a/packages/cua-anthropic/src/payload-hook.ts +++ b/packages/cua-anthropic/src/payload-hook.ts @@ -1,11 +1,11 @@ import type { Api, Model } from "@mariozechner/pi-ai"; -import { compactAnthropicMessagesForRequest } from "./context.js"; -import { anthropicComputerToolForModel } from "./official.js"; +import { compactAnthropicMessagesForRequest } from "./context"; +import { anthropicComputerToolForModel } from "./official"; import { ANTHROPIC_COMPACTION_EDIT_TYPE, ANTHROPIC_COMPACTION_MIN_TRIGGER_TOKENS, anthropicSupportsCompaction, -} from "./official.js"; +} from "./official"; /** * onPayload hook for Anthropic Messages requests. diff --git a/packages/cua-anthropic/src/pi/index.ts b/packages/cua-anthropic/src/pi/index.ts index 58d29d3a..c1709271 100644 --- a/packages/cua-anthropic/src/pi/index.ts +++ b/packages/cua-anthropic/src/pi/index.ts @@ -1,7 +1,7 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { ComputerTranslator } from "@onkernel/cua-translator"; -import { createAnthropicBatchTool } from "../batch-tool.js"; -import { createAnthropicComputerTool } from "../computer-tool.js"; +import { createAnthropicBatchTool } from "../batch-tool"; +import { createAnthropicComputerTool } from "../computer-tool"; export interface AnthropicComputerToolsOptions { /** Set false to omit the built-in `computer` tool. Defaults to true. */ @@ -24,17 +24,17 @@ export function createAnthropicComputerTools( return tools; } -export { createAnthropicComputerTool } from "../computer-tool.js"; -export type { AnthropicComputerDetails, AnthropicComputerInput } from "../computer.js"; -export { createAnthropicBatchTool } from "../batch-tool.js"; -export type { AnthropicBatchToolDetails, AnthropicBatchToolInput } from "../batch.js"; +export { createAnthropicComputerTool } from "../computer-tool"; +export type { AnthropicComputerDetails, AnthropicComputerInput } from "../computer"; +export { createAnthropicBatchTool } from "../batch-tool"; +export type { AnthropicBatchToolDetails, AnthropicBatchToolInput } from "../batch"; export { registerAnthropicProvider, wrapAnthropicStream, -} from "../stream-wrapper.js"; +} from "../stream-wrapper"; export { anthropicComputerOnPayload, composeOnPayload, createAnthropicContextManagementOnPayload, -} from "../payload-hook.js"; -export type { AnthropicContextManagementOptions } from "../payload-hook.js"; +} from "../payload-hook"; +export type { AnthropicContextManagementOptions } from "../payload-hook"; diff --git a/packages/cua-anthropic/src/stream-wrapper.ts b/packages/cua-anthropic/src/stream-wrapper.ts index 49b3aacb..29f427ff 100644 --- a/packages/cua-anthropic/src/stream-wrapper.ts +++ b/packages/cua-anthropic/src/stream-wrapper.ts @@ -8,8 +8,8 @@ import { ANTHROPIC_COMPACTION_BETA, anthropicComputerUseBetaForModel, anthropicSupportsCompaction, -} from "./official.js"; -import type { AnthropicContextManagementOptions } from "./payload-hook.js"; +} from "./official"; +import type { AnthropicContextManagementOptions } from "./payload-hook"; /** * Eagerly register the Anthropic Messages provider with `pi-ai`. pi-ai diff --git a/packages/cua-cli/src/action/result.ts b/packages/cua-cli/src/action/result.ts index dcaaad14..61b81e36 100644 --- a/packages/cua-cli/src/action/result.ts +++ b/packages/cua-cli/src/action/result.ts @@ -1,4 +1,4 @@ -import type { ActionType } from "./prompts.js"; +import type { ActionType } from "./prompts"; export type Status = "ok" | "not_found" | "error" | "timeout"; diff --git a/packages/cua-cli/src/action/runner.ts b/packages/cua-cli/src/action/runner.ts index dfef8bc4..ce9e93ec 100644 --- a/packages/cua-cli/src/action/runner.ts +++ b/packages/cua-cli/src/action/runner.ts @@ -3,12 +3,12 @@ import type { SessionManager } from "@mariozechner/pi-coding-agent"; import { type BrowserSession, ComputerTranslator } from "@onkernel/cua-translator"; import { writeFile } from "node:fs/promises"; import { stderr, stdout } from "node:process"; -import { promptWithScreenshot } from "../agent-prompt.js"; -import { type CuaAgentHandle, createCuaAgent } from "../agent.js"; -import type { Config } from "../config.js"; -import { persistAgentEvents, seedAgentFromSession } from "../sessions.js"; -import { type ActionRequest, buildPrompt, DEFAULT_MAX_TURNS } from "./prompts.js"; -import { type ActionEventInfo, type ActionResult, exitCodeFor, formatCompact, parseResult } from "./result.js"; +import { promptWithScreenshot } from "../agent-prompt"; +import { type CuaAgentHandle, createCuaAgent } from "../agent"; +import type { Config } from "../config"; +import { persistAgentEvents, seedAgentFromSession } from "../sessions"; +import { type ActionRequest, buildPrompt, DEFAULT_MAX_TURNS } from "./prompts"; +import { type ActionEventInfo, type ActionResult, exitCodeFor, formatCompact, parseResult } from "./result"; export interface RunOptions { cwd: string; diff --git a/packages/cua-cli/src/agent.ts b/packages/cua-cli/src/agent.ts index b06bde82..86c30d07 100644 --- a/packages/cua-cli/src/agent.ts +++ b/packages/cua-cli/src/agent.ts @@ -77,13 +77,13 @@ import { resolveOpenAIModelConfig, resolveTzafonModelConfig, resolveYutoriModelConfig, -} from "./config.js"; +} from "./config"; import { DEFAULT_MODEL_ID, type ProviderId, loadModel as loadSupportedModel, -} from "./models.js"; -import { appendSkillsToSystemPrompt, type Skill } from "./skills.js"; +} from "./models"; +import { appendSkillsToSystemPrompt, type Skill } from "./skills"; let providersRegistered = false; diff --git a/packages/cua-cli/src/cli.ts b/packages/cua-cli/src/cli.ts index d2ceadb5..4491cc83 100644 --- a/packages/cua-cli/src/cli.ts +++ b/packages/cua-cli/src/cli.ts @@ -2,18 +2,18 @@ import { browserSession, type BrowserSession } from "@onkernel/cua-translator"; import { stderr, stdout } from "node:process"; import { parseArgs } from "node:util"; -import { type ActionRequest, type ActionType } from "./action/prompts.js"; -import { emitCompact, runAction, type RunActionResult } from "./action/runner.js"; -import { createCuaAgent } from "./agent.js"; -import { promptWithScreenshot } from "./agent-prompt.js"; -import * as configMod from "./config.js"; +import { type ActionRequest, type ActionType } from "./action/prompts"; +import { emitCompact, runAction, type RunActionResult } from "./action/runner"; +import { createCuaAgent } from "./agent"; +import { promptWithScreenshot } from "./agent-prompt"; +import * as configMod from "./config"; import { DEFAULT_MODEL_ID, SUPPORTED_PROVIDERS, type ProviderId, listSupportedModels, resolveProvider, -} from "./models.js"; +} from "./models"; import { attachNamedSession, formatRelativeAge, @@ -24,8 +24,8 @@ import { startNamedSession, stopNamedSession, validateSlug, -} from "./named-sessions.js"; -import { attachJsonlSink } from "./output/jsonl.js"; +} from "./named-sessions"; +import { attachJsonlSink } from "./output/jsonl"; import { appendBrowserMetadata, findLatestSession, @@ -35,9 +35,9 @@ import { resolveSessionPath, seedAgentFromSession, type SessionInfo, -} from "./sessions.js"; -import { discoverCuaSkills, discoverStartupResources, expandSkillInvocation } from "./skills.js"; -import { runInteractive } from "./tui/main.js"; +} from "./sessions"; +import { discoverCuaSkills, discoverStartupResources, expandSkillInvocation } from "./skills"; +import { runInteractive } from "./tui/main"; const HELP = `cua — Kernel-cloud-browser computer-use agent diff --git a/packages/cua-cli/src/named-sessions.ts b/packages/cua-cli/src/named-sessions.ts index 11fb8662..576d9fb2 100644 --- a/packages/cua-cli/src/named-sessions.ts +++ b/packages/cua-cli/src/named-sessions.ts @@ -2,7 +2,7 @@ import { browserSession, type BrowserSession } from "@onkernel/cua-translator"; import { mkdir, readdir, readFile, stat, unlink, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { join } from "node:path"; -import type { Config } from "./config.js"; +import type { Config } from "./config"; /** * Named sessions: durable, slug-keyed pointers to a Kernel cloud browser diff --git a/packages/cua-cli/src/tui/driver.ts b/packages/cua-cli/src/tui/driver.ts index b94bd773..930a8b27 100644 --- a/packages/cua-cli/src/tui/driver.ts +++ b/packages/cua-cli/src/tui/driver.ts @@ -1,6 +1,6 @@ import type { Agent, AgentEvent } from "@mariozechner/pi-agent-core"; -import type { CuaAgentHandle } from "../agent.js"; -import { promptWithScreenshot } from "../agent-prompt.js"; +import type { CuaAgentHandle } from "../agent"; +import { promptWithScreenshot } from "../agent-prompt"; export type InteractiveDriverListener = (event: AgentEvent) => void | Promise; diff --git a/packages/cua-cli/src/tui/main.ts b/packages/cua-cli/src/tui/main.ts index f693db2c..fe781536 100644 --- a/packages/cua-cli/src/tui/main.ts +++ b/packages/cua-cli/src/tui/main.ts @@ -18,23 +18,23 @@ import type { BrowserSession } from "@onkernel/cua-translator"; import { homedir } from "node:os"; import { relative } from "node:path"; import { stderr } from "node:process"; -import { createCuaAgent } from "../agent.js"; -import type { Config } from "../config.js"; -import { DEFAULT_MODEL_ID, resolveProvider } from "../models.js"; +import { createCuaAgent } from "../agent"; +import type { Config } from "../config"; +import { DEFAULT_MODEL_ID, resolveProvider } from "../models"; import { appendBrowserMetadata, persistAgentEvents, seedAgentFromSession, -} from "../sessions.js"; -import { expandSkillInvocation, type Skill, type StartupResources } from "../skills.js"; -import { openTuiDebugLog } from "./debug-log.js"; -import { applyAndSummarizeImageProtocol } from "./diagnostics.js"; -import { LiveInteractiveDriver, type InteractiveDriver } from "./driver.js"; -import { type AssistantBuffer, MessageList } from "./message-list.js"; -import { ScreenshotWidget } from "./screenshot-widget.js"; -import { StatusLine } from "./status-line.js"; -import { TelemetryFooter } from "./telemetry-footer.js"; -import { colors, editorTheme } from "./themes.js"; +} from "../sessions"; +import { expandSkillInvocation, type Skill, type StartupResources } from "../skills"; +import { openTuiDebugLog } from "./debug-log"; +import { applyAndSummarizeImageProtocol } from "./diagnostics"; +import { LiveInteractiveDriver, type InteractiveDriver } from "./driver"; +import { type AssistantBuffer, MessageList } from "./message-list"; +import { ScreenshotWidget } from "./screenshot-widget"; +import { StatusLine } from "./status-line"; +import { TelemetryFooter } from "./telemetry-footer"; +import { colors, editorTheme } from "./themes"; export interface InteractiveOptions { cwd: string; diff --git a/packages/cua-cli/src/tui/message-list.ts b/packages/cua-cli/src/tui/message-list.ts index b6d8ec85..71bd3f7c 100644 --- a/packages/cua-cli/src/tui/message-list.ts +++ b/packages/cua-cli/src/tui/message-list.ts @@ -1,5 +1,5 @@ import { Container, Text } from "@mariozechner/pi-tui"; -import { colors } from "./themes.js"; +import { colors } from "./themes"; /** * Append-only chat log of user prompts, assistant text, tool-call summaries, diff --git a/packages/cua-cli/src/tui/screenshot-widget.ts b/packages/cua-cli/src/tui/screenshot-widget.ts index 9218b2de..e297ac9e 100644 --- a/packages/cua-cli/src/tui/screenshot-widget.ts +++ b/packages/cua-cli/src/tui/screenshot-widget.ts @@ -8,7 +8,7 @@ import { type ImageDimensions, type ImageTheme, } from "@mariozechner/pi-tui"; -import { imageTheme } from "./themes.js"; +import { imageTheme } from "./themes"; const MAX_WIDTH_CELLS = 60; diff --git a/packages/cua-cli/src/tui/status-line.ts b/packages/cua-cli/src/tui/status-line.ts index 929063b4..379e4ba0 100644 --- a/packages/cua-cli/src/tui/status-line.ts +++ b/packages/cua-cli/src/tui/status-line.ts @@ -1,5 +1,5 @@ import { Text, hyperlink } from "@mariozechner/pi-tui"; -import { colors } from "./themes.js"; +import { colors } from "./themes"; export interface StatusLineState { model: string; diff --git a/packages/cua-cli/src/tui/telemetry-footer.ts b/packages/cua-cli/src/tui/telemetry-footer.ts index 6b31500d..f3affada 100644 --- a/packages/cua-cli/src/tui/telemetry-footer.ts +++ b/packages/cua-cli/src/tui/telemetry-footer.ts @@ -1,5 +1,5 @@ import { type Component, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; -import { colors } from "./themes.js"; +import { colors } from "./themes"; export interface TelemetryFooterState { provider?: string; diff --git a/packages/cua-cli/src/tui/testing/fixture-main.ts b/packages/cua-cli/src/tui/testing/fixture-main.ts index c75e6fa3..7c4cfb03 100644 --- a/packages/cua-cli/src/tui/testing/fixture-main.ts +++ b/packages/cua-cli/src/tui/testing/fixture-main.ts @@ -1,9 +1,9 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import type { BrowserSession } from "@onkernel/cua-translator"; -import type { Config } from "../../config.js"; -import { runInteractive } from "../main.js"; -import { ScriptedDriver, type ScriptedFixture } from "./scripted-driver.js"; +import type { Config } from "../../config"; +import { runInteractive } from "../main"; +import { ScriptedDriver, type ScriptedFixture } from "./scripted-driver"; async function main(): Promise { const fixtureArg = process.argv[2]; diff --git a/packages/cua-cli/src/tui/testing/scripted-driver.ts b/packages/cua-cli/src/tui/testing/scripted-driver.ts index 93c4089d..b471ec5b 100644 --- a/packages/cua-cli/src/tui/testing/scripted-driver.ts +++ b/packages/cua-cli/src/tui/testing/scripted-driver.ts @@ -1,6 +1,6 @@ import type { AgentEvent } from "@mariozechner/pi-agent-core"; import type { AssistantMessage, Usage } from "@mariozechner/pi-ai"; -import type { InteractiveDriver, InteractiveDriverListener } from "../driver.js"; +import type { InteractiveDriver, InteractiveDriverListener } from "../driver"; export interface FixturePromptMatch { equals?: string; diff --git a/packages/cua-gemini/src/computer-tool.ts b/packages/cua-gemini/src/computer-tool.ts index 4ea99d27..ba78c46e 100644 --- a/packages/cua-gemini/src/computer-tool.ts +++ b/packages/cua-gemini/src/computer-tool.ts @@ -7,8 +7,8 @@ import { forwardModelAction, } from "@onkernel/cua-translator"; import { type Static, Type } from "@sinclair/typebox"; -import { denormalizeX, denormalizeY } from "./coords.js"; -import { DEFAULT_GEMINI_SCREEN_SIZE, GeminiAction, type GeminiScreenSize } from "./official.js"; +import { denormalizeX, denormalizeY } from "./coords"; +import { DEFAULT_GEMINI_SCREEN_SIZE, GeminiAction, type GeminiScreenSize } from "./official"; /** * One AgentTool per predefined Gemini computer-use function. diff --git a/packages/cua-gemini/src/computer.ts b/packages/cua-gemini/src/computer.ts index d261c6da..287283ad 100644 --- a/packages/cua-gemini/src/computer.ts +++ b/packages/cua-gemini/src/computer.ts @@ -8,12 +8,12 @@ import { forwardModelAction, } from "@onkernel/cua-translator"; import { type TObject, Type } from "@sinclair/typebox"; -import { denormalizeX, denormalizeY } from "./coords.js"; +import { denormalizeX, denormalizeY } from "./coords"; import { DEFAULT_GEMINI_SCREEN_SIZE, GeminiAction, type GeminiScreenSize, -} from "./official.js"; +} from "./official"; const SCREENSHOT_DELAY_MS = 500; const PX_PER_NOTCH = 60; diff --git a/packages/cua-gemini/src/coords.ts b/packages/cua-gemini/src/coords.ts index d6049d9d..961ae65f 100644 --- a/packages/cua-gemini/src/coords.ts +++ b/packages/cua-gemini/src/coords.ts @@ -1,4 +1,4 @@ -import { GEMINI_COORDINATE_SCALE, type GeminiScreenSize } from "./official.js"; +import { GEMINI_COORDINATE_SCALE, type GeminiScreenSize } from "./official"; /** Convert Gemini's 0–1000 normalized X to pixels for the given screen. */ export function denormalizeX(x: number, screen: GeminiScreenSize): number { diff --git a/packages/cua-gemini/src/index.ts b/packages/cua-gemini/src/index.ts index 0474fd47..dca7d47a 100644 --- a/packages/cua-gemini/src/index.ts +++ b/packages/cua-gemini/src/index.ts @@ -18,34 +18,34 @@ export { GEMINI_BATCH_DESCRIPTION, GEMINI_BATCH_FUNCTION_DECLARATION, GEMINI_BATCH_TOOL_NAME, -} from "./batch.js"; -export type { GeminiBatchToolInput, GeminiBatchToolDetails } from "./batch.js"; +} from "./batch"; +export type { GeminiBatchToolInput, GeminiBatchToolDetails } from "./batch"; export { executeGeminiFunctionCall, GEMINI_FUNCTION_DECLARATIONS, -} from "./computer.js"; -export type { GeminiComputerToolsOptions, GeminiToolDetails } from "./computer.js"; +} from "./computer"; +export type { GeminiComputerToolsOptions, GeminiToolDetails } from "./computer"; export { GeminiAction, PREDEFINED_COMPUTER_USE_FUNCTIONS, DEFAULT_GEMINI_SCREEN_SIZE, GEMINI_COORDINATE_SCALE, -} from "./official.js"; +} from "./official"; export type { GeminiFunctionArgs, GeminiScreenSize, ScrollDirection, -} from "./official.js"; -export { gemini } from "./model.js"; +} from "./official"; +export { gemini } from "./model"; export type { GeminiModelOptions, GeminiModelRunDetails, -} from "./model.js"; -export { denormalizeX, denormalizeY } from "./coords.js"; -export { GEMINI_CUA_EXTRA_ACTION_TYPES } from "./cua-extras.js"; -export type { GeminiCuaExtraActionType } from "./cua-extras.js"; +} from "./model"; +export { denormalizeX, denormalizeY } from "./coords"; +export { GEMINI_CUA_EXTRA_ACTION_TYPES } from "./cua-extras"; +export type { GeminiCuaExtraActionType } from "./cua-extras"; export { buildGeminiSystemPrompt, GEMINI_INSTRUCTIONS_RAW, -} from "./system-prompt.js"; -export type { GeminiSystemPromptOptions } from "./system-prompt.js"; +} from "./system-prompt"; +export type { GeminiSystemPromptOptions } from "./system-prompt"; diff --git a/packages/cua-gemini/src/model.ts b/packages/cua-gemini/src/model.ts index 6ed1ec0e..f0234a82 100644 --- a/packages/cua-gemini/src/model.ts +++ b/packages/cua-gemini/src/model.ts @@ -8,14 +8,14 @@ import { executeGeminiBatch, GEMINI_BATCH_FUNCTION_DECLARATION, GEMINI_BATCH_TOOL_NAME, -} from "./batch.js"; +} from "./batch"; import { type GeminiComputerToolsOptions, type GeminiToolDetails, executeGeminiFunctionCall, GEMINI_FUNCTION_DECLARATIONS, -} from "./computer.js"; -import { buildGeminiSystemPrompt } from "./system-prompt.js"; +} from "./computer"; +import { buildGeminiSystemPrompt } from "./system-prompt"; export interface GeminiModelOptions extends GeminiComputerToolsOptions { apiKey?: string; diff --git a/packages/cua-gemini/src/pi/index.ts b/packages/cua-gemini/src/pi/index.ts index dca65ac0..cf682d7a 100644 --- a/packages/cua-gemini/src/pi/index.ts +++ b/packages/cua-gemini/src/pi/index.ts @@ -1,10 +1,10 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { ComputerTranslator } from "@onkernel/cua-translator"; -import { createGeminiBatchTool } from "../batch-tool.js"; +import { createGeminiBatchTool } from "../batch-tool"; import { createGeminiPerActionTools, type GeminiComputerToolsOptions as PerActionOptions, -} from "../computer-tool.js"; +} from "../computer-tool"; export interface GeminiComputerToolsOptions extends PerActionOptions { includeBatch?: boolean; @@ -24,7 +24,7 @@ export function createGeminiComputerTools( return tools; } -export { createGeminiPerActionTools } from "../computer-tool.js"; -export type { GeminiToolDetails } from "../computer-tool.js"; -export { createGeminiBatchTool } from "../batch-tool.js"; -export type { GeminiBatchToolInput, GeminiBatchToolDetails } from "../batch.js"; +export { createGeminiPerActionTools } from "../computer-tool"; +export type { GeminiToolDetails } from "../computer-tool"; +export { createGeminiBatchTool } from "../batch-tool"; +export type { GeminiBatchToolInput, GeminiBatchToolDetails } from "../batch"; diff --git a/packages/cua-openai/src/batch-tool.ts b/packages/cua-openai/src/batch-tool.ts index 2c64d91b..00ee9edb 100644 --- a/packages/cua-openai/src/batch-tool.ts +++ b/packages/cua-openai/src/batch-tool.ts @@ -7,7 +7,7 @@ import { executeOpenAIBatch, OPENAI_BATCH_DESCRIPTION, OPENAI_BATCH_TOOL_NAME, -} from "./batch.js"; +} from "./batch"; /** * Build the `batch_computer_actions` AgentTool for OpenAI computer-use models. diff --git a/packages/cua-openai/src/batch.ts b/packages/cua-openai/src/batch.ts index d784dff1..1a81b776 100644 --- a/packages/cua-openai/src/batch.ts +++ b/packages/cua-openai/src/batch.ts @@ -1,7 +1,7 @@ import type { ComputerTranslator, ComputerUseToolResult, ModelAction } from "@onkernel/cua-translator"; import { type Static, Type } from "@sinclair/typebox"; -import { OPENAI_CUA_EXTRA_ACTION_TYPES } from "./cua-extras.js"; -import { OPENAI_OFFICIAL_ACTION_TYPES } from "./official.js"; +import { OPENAI_CUA_EXTRA_ACTION_TYPES } from "./cua-extras"; +import { OPENAI_OFFICIAL_ACTION_TYPES } from "./official"; export const OPENAI_BATCH_TOOL_NAME = "batch_computer_actions"; diff --git a/packages/cua-openai/src/extra-tool.ts b/packages/cua-openai/src/extra-tool.ts index 4624d072..9060e047 100644 --- a/packages/cua-openai/src/extra-tool.ts +++ b/packages/cua-openai/src/extra-tool.ts @@ -7,7 +7,7 @@ import { type ExtraToolDetails, OPENAI_EXTRA_TOOL_DESCRIPTION, OPENAI_EXTRA_TOOL_NAME, -} from "./extra.js"; +} from "./extra"; /** * Build the `computer_use_extra` AgentTool. High-level browser navigation diff --git a/packages/cua-openai/src/index.ts b/packages/cua-openai/src/index.ts index bd8679f2..20d68bd6 100644 --- a/packages/cua-openai/src/index.ts +++ b/packages/cua-openai/src/index.ts @@ -17,39 +17,39 @@ export { OPENAI_BATCH_DESCRIPTION, OPENAI_BATCH_TOOL, OPENAI_BATCH_TOOL_NAME, -} from "./batch.js"; -export type { BatchToolDetails, BatchToolInput } from "./batch.js"; +} from "./batch"; +export type { BatchToolDetails, BatchToolInput } from "./batch"; export { executeOpenAIExtraAction, ExtraSchema, OPENAI_EXTRA_TOOL, OPENAI_EXTRA_TOOL_DESCRIPTION, OPENAI_EXTRA_TOOL_NAME, -} from "./extra.js"; -export type { ExtraToolDetails, ExtraToolInput } from "./extra.js"; +} from "./extra"; +export type { ExtraToolDetails, ExtraToolInput } from "./extra"; export { OPENAI_OFFICIAL_ACTION_TYPES, OpenAIOfficialActionSchema, OpenAIPointSchema, -} from "./official.js"; -export type { OpenAIOfficialAction, OpenAIOfficialActionType, OpenAIPoint } from "./official.js"; +} from "./official"; +export type { OpenAIOfficialAction, OpenAIOfficialActionType, OpenAIPoint } from "./official"; export { OPENAI_CUA_EXTRA_ACTION_TYPES, OpenAICuaExtraActionSchema, -} from "./cua-extras.js"; -export type { OpenAICuaExtraAction, OpenAICuaExtraActionType } from "./cua-extras.js"; +} from "./cua-extras"; +export type { OpenAICuaExtraAction, OpenAICuaExtraActionType } from "./cua-extras"; export { executeOpenAIToolCall, openai, openaiTools, -} from "./model.js"; +} from "./model"; export type { OpenAIModelOptions, OpenAIModelRunDetails, OpenAIToolCallResultDetails, OpenAIToolSpec, -} from "./model.js"; +} from "./model"; export { OPENAI_BATCH_INSTRUCTIONS, OPENAI_NATIVE_COMPUTER_INSTRUCTIONS, -} from "./system-prompt.js"; +} from "./system-prompt"; diff --git a/packages/cua-openai/src/model.ts b/packages/cua-openai/src/model.ts index 5e98d2b7..9b1f0758 100644 --- a/packages/cua-openai/src/model.ts +++ b/packages/cua-openai/src/model.ts @@ -17,9 +17,9 @@ import type { ResponseOutputMessage, Tool, } from "openai/resources/responses/responses"; -import { type BatchToolDetails, type BatchToolInput, executeOpenAIBatch, OPENAI_BATCH_TOOL } from "./batch.js"; -import { type ExtraToolDetails, type ExtraToolInput, executeOpenAIExtraAction, OPENAI_EXTRA_TOOL } from "./extra.js"; -import { OPENAI_BATCH_INSTRUCTIONS, OPENAI_NATIVE_COMPUTER_INSTRUCTIONS } from "./system-prompt.js"; +import { type BatchToolDetails, type BatchToolInput, executeOpenAIBatch, OPENAI_BATCH_TOOL } from "./batch"; +import { type ExtraToolDetails, type ExtraToolInput, executeOpenAIExtraAction, OPENAI_EXTRA_TOOL } from "./extra"; +import { OPENAI_BATCH_INSTRUCTIONS, OPENAI_NATIVE_COMPUTER_INSTRUCTIONS } from "./system-prompt"; const OPENAI_COMPUTER_TOOL = { type: "computer" } as const; const POST_ACTION_SETTLE_MS = 300; diff --git a/packages/cua-openai/src/pi/index.ts b/packages/cua-openai/src/pi/index.ts index 4542264e..e76c011b 100644 --- a/packages/cua-openai/src/pi/index.ts +++ b/packages/cua-openai/src/pi/index.ts @@ -1,7 +1,7 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { ComputerTranslator } from "@onkernel/cua-translator"; -import { createBatchTool } from "../batch-tool.js"; -import { createExtraTool } from "../extra-tool.js"; +import { createBatchTool } from "../batch-tool"; +import { createExtraTool } from "../extra-tool"; export interface OpenAIComputerToolsOptions { /** Set false to omit `batch_computer_actions`. Defaults to true. */ @@ -24,7 +24,7 @@ export function createOpenAIComputerTools( return tools; } -export { createBatchTool } from "../batch-tool.js"; -export type { BatchToolDetails, BatchToolInput } from "../batch.js"; -export { createExtraTool } from "../extra-tool.js"; -export type { ExtraToolDetails, ExtraToolInput } from "../extra.js"; +export { createBatchTool } from "../batch-tool"; +export type { BatchToolDetails, BatchToolInput } from "../batch"; +export { createExtraTool } from "../extra-tool"; +export type { ExtraToolDetails, ExtraToolInput } from "../extra"; diff --git a/packages/cua-translator/src/computer-use.ts b/packages/cua-translator/src/computer-use.ts index 7df36247..763d623a 100644 --- a/packages/cua-translator/src/computer-use.ts +++ b/packages/cua-translator/src/computer-use.ts @@ -1,5 +1,5 @@ -import type { BrowserSession } from "./browser-session.js"; -import { ComputerTranslator, type ComputerLogger } from "./translator.js"; +import type { BrowserSession } from "./browser-session"; +import { ComputerTranslator, type ComputerLogger } from "./translator"; export interface ComputerUseTextPart { type: "text"; diff --git a/packages/cua-translator/src/cua-extras.ts b/packages/cua-translator/src/cua-extras.ts index 4426ff5a..4b574b78 100644 --- a/packages/cua-translator/src/cua-extras.ts +++ b/packages/cua-translator/src/cua-extras.ts @@ -16,8 +16,8 @@ * talking to the high-level translator API. */ -import { translateKeys } from "./keysym.js"; -import type { BatchAction, ModelAction } from "./types.js"; +import { translateKeys } from "./keysym"; +import type { BatchAction, ModelAction } from "./types"; // ─── BatchAction-level builders ──────────────────────────────────────────── diff --git a/packages/cua-translator/src/index.ts b/packages/cua-translator/src/index.ts index 18a89363..b8b7ebef 100644 --- a/packages/cua-translator/src/index.ts +++ b/packages/cua-translator/src/index.ts @@ -1,12 +1,12 @@ -export * from "./types.js"; -export * from "./keysym.js"; -export * from "./scroll.js"; -export * from "./cua-extras.js"; -export * from "./translator.js"; -export * from "./computer-use.js"; -export * as browserSession from "./browser-session.js"; +export * from "./types"; +export * from "./keysym"; +export * from "./scroll"; +export * from "./cua-extras"; +export * from "./translator"; +export * from "./computer-use"; +export * as browserSession from "./browser-session"; export type { AttachBrowserSessionOptions, BrowserSession, BrowserSessionOptions, -} from "./browser-session.js"; +} from "./browser-session"; diff --git a/packages/cua-translator/src/translator.ts b/packages/cua-translator/src/translator.ts index 091443dc..44ec3b20 100644 --- a/packages/cua-translator/src/translator.ts +++ b/packages/cua-translator/src/translator.ts @@ -4,9 +4,9 @@ import { currentUrlCopyActions, forwardBatchActions, gotoBatchActions, -} from "./cua-extras.js"; -import { splitKeypress, translateKeys } from "./keysym.js"; -import { modelScrollDeltaToWheelTicks } from "./scroll.js"; +} from "./cua-extras"; +import { splitKeypress, translateKeys } from "./keysym"; +import { modelScrollDeltaToWheelTicks } from "./scroll"; import { type ActionValidationScope, ActionValidationError, @@ -14,7 +14,7 @@ import { type BatchAction, type BatchExecutionResult, type ModelAction, -} from "./types.js"; +} from "./types"; // ─── Numeric helpers ─────────────────────────────────────────────────────── @@ -208,7 +208,7 @@ export function translateToBatchAction(actionType: string, action: ModelAction, // ─── SDK conversion ──────────────────────────────────────────────────────── -type SdkBatchAction = import("@onkernel/sdk/resources/browsers/computer.js").ComputerBatchParams.Action; +type SdkBatchAction = import("@onkernel/sdk/resources/browsers/computer").ComputerBatchParams.Action; export function toSdkAction(action: BatchAction): SdkBatchAction { switch (action.type) { diff --git a/packages/cua-tzafon/src/computer.ts b/packages/cua-tzafon/src/computer.ts index 2c9a53b1..ff2b993e 100644 --- a/packages/cua-tzafon/src/computer.ts +++ b/packages/cua-tzafon/src/computer.ts @@ -4,12 +4,12 @@ import type { ModelAction, } from "@onkernel/cua-translator"; import { type Static, Type } from "@sinclair/typebox"; -import { denormalizeX, denormalizeY } from "./coords.js"; +import { denormalizeX, denormalizeY } from "./coords"; import { DEFAULT_TZAFON_SCREEN_SIZE, TzafonAction, type TzafonScreenSize, -} from "./official.js"; +} from "./official"; const SCREENSHOT_DELAY_MS = 500; const SCROLL_WHEEL_TICKS_PER_NOTCH = 120; diff --git a/packages/cua-tzafon/src/coords.ts b/packages/cua-tzafon/src/coords.ts index de56fbee..638d2dea 100644 --- a/packages/cua-tzafon/src/coords.ts +++ b/packages/cua-tzafon/src/coords.ts @@ -2,7 +2,7 @@ import { DEFAULT_TZAFON_SCREEN_SIZE, TZAFON_COORDINATE_SCALE, type TzafonScreenSize, -} from "./official.js"; +} from "./official"; export function parseTzafonCoord(value: unknown): number { if (value == null) return 0; diff --git a/packages/cua-tzafon/src/index.ts b/packages/cua-tzafon/src/index.ts index 0801ce19..0c157afc 100644 --- a/packages/cua-tzafon/src/index.ts +++ b/packages/cua-tzafon/src/index.ts @@ -11,35 +11,35 @@ export { denormalizeX, denormalizeY, parseTzafonCoord, -} from "./coords.js"; +} from "./coords"; export { executeTzafonFunctionCall, getTzafonDefinition, splitKeyCombo, TZAFON_FUNCTION_TOOLS, -} from "./computer.js"; +} from "./computer"; export type { TzafonComputerToolsOptions, TzafonToolDetails, -} from "./computer.js"; +} from "./computer"; export { TzafonAction, DEFAULT_TZAFON_SCREEN_SIZE, TZAFON_ACTIONS, TZAFON_COORDINATE_SCALE, TZAFON_DEFAULT_MODEL, -} from "./official.js"; +} from "./official"; export type { TzafonActionName, TzafonScreenSize, -} from "./official.js"; -export { tzafon } from "./model.js"; +} from "./official"; +export { tzafon } from "./model"; export type { TzafonModelOptions, TzafonModelRunDetails, -} from "./model.js"; +} from "./model"; export { buildTzafonSystemPrompt, TZAFON_INSTRUCTIONS_RAW, -} from "./system-prompt.js"; -export type { TzafonSystemPromptOptions } from "./system-prompt.js"; +} from "./system-prompt"; +export type { TzafonSystemPromptOptions } from "./system-prompt"; diff --git a/packages/cua-tzafon/src/model.ts b/packages/cua-tzafon/src/model.ts index b103b2a0..60706d2b 100644 --- a/packages/cua-tzafon/src/model.ts +++ b/packages/cua-tzafon/src/model.ts @@ -10,9 +10,9 @@ import { TZAFON_FUNCTION_TOOLS, type TzafonComputerToolsOptions, type TzafonToolDetails, -} from "./computer.js"; -import { TZAFON_DEFAULT_MODEL } from "./official.js"; -import { buildTzafonSystemPrompt } from "./system-prompt.js"; +} from "./computer"; +import { TZAFON_DEFAULT_MODEL } from "./official"; +import { buildTzafonSystemPrompt } from "./system-prompt"; const DEFAULT_MAX_OUTPUT_TOKENS = 4096; const MAX_HISTORY_ITEMS = 30; diff --git a/packages/cua-tzafon/src/pi/index.ts b/packages/cua-tzafon/src/pi/index.ts index 23306bbb..67569bc3 100644 --- a/packages/cua-tzafon/src/pi/index.ts +++ b/packages/cua-tzafon/src/pi/index.ts @@ -21,7 +21,7 @@ import { TZAFON_FUNCTION_TOOLS, type TzafonComputerToolsOptions, type TzafonToolDetails, -} from "../computer.js"; +} from "../computer"; export const TZAFON_RESPONSES_API = "tzafon-responses"; diff --git a/packages/cua-yutori/src/computer-tool.ts b/packages/cua-yutori/src/computer-tool.ts index 47253e0c..fc8e64c2 100644 --- a/packages/cua-yutori/src/computer-tool.ts +++ b/packages/cua-yutori/src/computer-tool.ts @@ -7,7 +7,7 @@ import { type YutoriComputerToolsOptions, type YutoriToolDetails, executeYutoriFunctionCall, -} from "./computer.js"; +} from "./computer"; export function createYutoriPerActionTools( translator: ComputerTranslator, diff --git a/packages/cua-yutori/src/computer.ts b/packages/cua-yutori/src/computer.ts index b0ce88cb..96d8cfb7 100644 --- a/packages/cua-yutori/src/computer.ts +++ b/packages/cua-yutori/src/computer.ts @@ -5,13 +5,13 @@ import { backModelAction, } from "@onkernel/cua-translator"; import { type TObject, Type } from "@sinclair/typebox"; -import { denormalizeX, denormalizeY } from "./coords.js"; +import { denormalizeX, denormalizeY } from "./coords"; import { DEFAULT_YUTORI_SCREEN_SIZE, YutoriAction, type YutoriScreenSize, type YutoriScrollDirection, -} from "./official.js"; +} from "./official"; const SCREENSHOT_DELAY_MS = 300; const WAIT_MS = 2000; diff --git a/packages/cua-yutori/src/coords.ts b/packages/cua-yutori/src/coords.ts index 9c6c4931..6dab57ea 100644 --- a/packages/cua-yutori/src/coords.ts +++ b/packages/cua-yutori/src/coords.ts @@ -1,4 +1,4 @@ -import { YUTORI_COORDINATE_SCALE, type YutoriScreenSize } from "./official.js"; +import { YUTORI_COORDINATE_SCALE, type YutoriScreenSize } from "./official"; export function denormalizeX(x: number, screen: YutoriScreenSize): number { return Math.round((x / YUTORI_COORDINATE_SCALE) * screen.width); diff --git a/packages/cua-yutori/src/index.ts b/packages/cua-yutori/src/index.ts index 7d9cd099..380cefa2 100644 --- a/packages/cua-yutori/src/index.ts +++ b/packages/cua-yutori/src/index.ts @@ -11,26 +11,26 @@ export { executeYutoriFunctionCall, YUTORI_DEFINITIONS, YUTORI_FUNCTION_DECLARATIONS, -} from "./computer.js"; +} from "./computer"; export type { YutoriComputerToolsOptions, YutoriToolDetails, -} from "./computer.js"; +} from "./computer"; export { DEFAULT_YUTORI_SCREEN_SIZE, YUTORI_ACTION_TYPES, YUTORI_COORDINATE_SCALE, YUTORI_MODEL_IDS, YutoriAction, -} from "./official.js"; +} from "./official"; export type { YutoriActionType, YutoriModelId, YutoriScreenSize, YutoriScrollDirection, -} from "./official.js"; -export { denormalizeX, denormalizeY } from "./coords.js"; -export { yutori } from "./model.js"; -export type { YutoriModelOptions, YutoriModelRunDetails } from "./model.js"; -export { buildYutoriSystemPrompt } from "./system-prompt.js"; -export type { YutoriSystemPromptOptions } from "./system-prompt.js"; +} from "./official"; +export { denormalizeX, denormalizeY } from "./coords"; +export { yutori } from "./model"; +export type { YutoriModelOptions, YutoriModelRunDetails } from "./model"; +export { buildYutoriSystemPrompt } from "./system-prompt"; +export type { YutoriSystemPromptOptions } from "./system-prompt"; diff --git a/packages/cua-yutori/src/model.ts b/packages/cua-yutori/src/model.ts index faa1e789..6245aef1 100644 --- a/packages/cua-yutori/src/model.ts +++ b/packages/cua-yutori/src/model.ts @@ -4,7 +4,7 @@ import type { } from "@onkernel/cua-translator"; import OpenAI from "openai"; import type { ChatCompletionMessageParam } from "openai/resources/chat/completions"; -import { executeYutoriFunctionCall, type YutoriComputerToolsOptions } from "./computer.js"; +import { executeYutoriFunctionCall, type YutoriComputerToolsOptions } from "./computer"; export interface YutoriModelOptions extends YutoriComputerToolsOptions { apiKey?: string; diff --git a/packages/cua-yutori/src/pi/index.ts b/packages/cua-yutori/src/pi/index.ts index 29a41aea..219933f4 100644 --- a/packages/cua-yutori/src/pi/index.ts +++ b/packages/cua-yutori/src/pi/index.ts @@ -1,15 +1,15 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; import type { ComputerTranslator } from "@onkernel/cua-translator"; -import { createYutoriPerActionTools } from "../computer-tool.js"; -import type { YutoriComputerToolsOptions } from "../computer.js"; -import { YUTORI_ACTION_TYPES } from "../official.js"; +import { createYutoriPerActionTools } from "../computer-tool"; +import type { YutoriComputerToolsOptions } from "../computer"; +import { YUTORI_ACTION_TYPES } from "../official"; export { YUTORI_CHAT_COMPLETIONS_API, registerYutoriProvider, streamSimpleYutori, streamYutori, -} from "./provider.js"; -export type { YutoriOptions } from "./provider.js"; +} from "./provider"; +export type { YutoriOptions } from "./provider"; const YUTORI_BUILTIN_TOOL_NAMES = new Set(YUTORI_ACTION_TYPES); @@ -47,5 +47,5 @@ function readToolName(tool: unknown): string | undefined { return undefined; } -export { createYutoriPerActionTools } from "../computer-tool.js"; -export type { YutoriToolDetails, YutoriComputerToolsOptions } from "../computer.js"; +export { createYutoriPerActionTools } from "../computer-tool"; +export type { YutoriToolDetails, YutoriComputerToolsOptions } from "../computer"; diff --git a/packages/cua-yutori/src/pi/provider.ts b/packages/cua-yutori/src/pi/provider.ts index 5d730a69..b4ebf0dd 100644 --- a/packages/cua-yutori/src/pi/provider.ts +++ b/packages/cua-yutori/src/pi/provider.ts @@ -14,7 +14,7 @@ import { createAssistantMessageEventStream, registerApiProvider, } from "@mariozechner/pi-ai"; -import { YUTORI_ACTION_TYPES } from "../official.js"; +import { YUTORI_ACTION_TYPES } from "../official"; export const YUTORI_CHAT_COMPLETIONS_API = "yutori-chat-completions"; diff --git a/packages/ptywright/src/index.ts b/packages/ptywright/src/index.ts index 1028c292..05a21f6a 100644 --- a/packages/ptywright/src/index.ts +++ b/packages/ptywright/src/index.ts @@ -1,3 +1,3 @@ -export * from "./keys.js"; -export * from "./session.js"; -export * from "./terminal.js"; +export * from "./keys"; +export * from "./session"; +export * from "./terminal"; diff --git a/packages/ptywright/src/session.ts b/packages/ptywright/src/session.ts index e452cc32..62eb7e70 100644 --- a/packages/ptywright/src/session.ts +++ b/packages/ptywright/src/session.ts @@ -1,8 +1,8 @@ import { EventEmitter } from "node:events"; import { mkdir, writeFile } from "node:fs/promises"; import { spawn, type IPty } from "node-pty"; -import { KeyEnter, type Key } from "./keys.js"; -import { createTerminal, type SnapshotOptions, type TerminalSnapshot, type TerminalSurface } from "./terminal.js"; +import { KeyEnter, type Key } from "./keys"; +import { createTerminal, type SnapshotOptions, type TerminalSnapshot, type TerminalSurface } from "./terminal"; const DEFAULT_COLS = 120; const DEFAULT_ROWS = 40; diff --git a/packages/ptywright/src/terminal.ts b/packages/ptywright/src/terminal.ts index 13abe68e..1ece2a5d 100644 --- a/packages/ptywright/src/terminal.ts +++ b/packages/ptywright/src/terminal.ts @@ -1,4 +1,4 @@ -import { loadNativeBinding, type NativeSnapshot, type NativeTerminalHandle } from "./native-loader.js"; +import { loadNativeBinding, type NativeSnapshot, type NativeTerminalHandle } from "./native-loader"; export interface CreateTerminalOptions { cols: number; diff --git a/packages/ptywright/src/test/session.test.ts b/packages/ptywright/src/test/session.test.ts index 98b905c2..7e7bc67b 100644 --- a/packages/ptywright/src/test/session.test.ts +++ b/packages/ptywright/src/test/session.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { KeyCtrlD, spawnSession } from "../index.js"; +import { KeyCtrlD, spawnSession } from "../index"; test("session captures transcript and visible screen", async (t) => { const session = spawnSession({ diff --git a/packages/ptywright/src/test/terminal.test.ts b/packages/ptywright/src/test/terminal.test.ts index 4c2f024d..df01634c 100644 --- a/packages/ptywright/src/test/terminal.test.ts +++ b/packages/ptywright/src/test/terminal.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { createTerminal } from "../terminal.js"; +import { createTerminal } from "../terminal"; test("terminal snapshots formatted visible text", () => { const terminal = createTerminal({ cols: 40, rows: 6 });