diff --git a/front/lib/api/llm/transitionLLM.test.ts b/front/lib/api/llm/transitionLLM.test.ts index 27fffe4634a6..8bac6ecd76f2 100644 --- a/front/lib/api/llm/transitionLLM.test.ts +++ b/front/lib/api/llm/transitionLLM.test.ts @@ -37,9 +37,11 @@ const serverToolUseBlock = { function reasoningMessage({ provider, metadata, + reasoning = "let me think", }: { provider: ModelProviderIdType; metadata: string; + reasoning?: string; }) { return { role: "assistant" as const, @@ -48,7 +50,7 @@ function reasoningMessage({ { type: "reasoning" as const, value: { - reasoning: "let me think", + reasoning, metadata, tokens: 10, provider, @@ -240,6 +242,23 @@ describe("toBaseMessages — reasoning signatures", () => { }, ]); }); + + it("keeps a Fireworks Responses reasoning id in `signature`", () => { + const result = toBaseMessages( + reasoningMessage({ + provider: "fireworks", + metadata: JSON.stringify({ id: "rs_fireworks_123" }), + }) + ); + expect(result).toEqual([ + { + role: "assistant", + type: "reasoning", + content: { value: "let me think" }, + signature: "rs_fireworks_123", + }, + ]); + }); }); describe("OpenAI reasoning round-trip — persisted metadata to Responses input", () => { @@ -267,6 +286,109 @@ describe("OpenAI reasoning round-trip — persisted metadata to Responses input" }); }); +describe("Fireworks reasoning round-trip — persisted metadata to Responses input", () => { + it("replays an id-bearing reasoning item with an empty summary", () => { + const [baseMessage] = toBaseMessages( + reasoningMessage({ + provider: "fireworks", + metadata: JSON.stringify({ id: "rs_fireworks_empty" }), + reasoning: "", + }) + ); + if (baseMessage.type !== "reasoning") { + throw new Error("Expected a reasoning BaseMessage."); + } + + expect(assistantReasoningMessageToInputItems(baseMessage)).toEqual([ + { + id: "rs_fireworks_empty", + type: "reasoning", + summary: [], + }, + ]); + }); + + it("preserves interleaved reasoning ids and tool calls in wire order", () => { + const message: ModelMessageTypeMultiActionsWithoutContentFragment = { + role: "assistant", + name: "agent", + contents: [ + { + type: "reasoning", + value: { + reasoning: "first thought", + metadata: JSON.stringify({ id: "rs_fireworks_1" }), + tokens: 10, + provider: "fireworks", + }, + }, + { + type: "function_call", + value: { + id: "call_1", + name: "first_tool", + arguments: "{}", + }, + }, + { + type: "reasoning", + value: { + reasoning: "second thought", + metadata: JSON.stringify({ id: "rs_fireworks_2" }), + tokens: 10, + provider: "fireworks", + }, + }, + { + type: "function_call", + value: { + id: "call_2", + name: "second_tool", + arguments: "{}", + }, + }, + ], + }; + + expect(toBaseMessages(message)).toEqual([ + { + role: "assistant", + type: "reasoning", + content: { value: "first thought" }, + signature: "rs_fireworks_1", + }, + { + role: "assistant", + type: "tool_call_request", + content: { + callId: "call_1", + toolName: "first_tool", + arguments: "{}", + namespace: undefined, + }, + signature: undefined, + }, + { + role: "assistant", + type: "reasoning", + content: { value: "second thought" }, + signature: "rs_fireworks_2", + }, + { + role: "assistant", + type: "tool_call_request", + content: { + callId: "call_2", + toolName: "second_tool", + arguments: "{}", + namespace: undefined, + }, + signature: undefined, + }, + ]); + }); +}); + describe("convertToOldEvent — token_usage", () => { it("sums the per-TTL cache-creation breakdown into cacheCreationTokens and keeps the split", () => { expect( diff --git a/front/lib/api/llm/transitionLLM.ts b/front/lib/api/llm/transitionLLM.ts index ba60e7f5c836..4a90d2f69bbb 100644 --- a/front/lib/api/llm/transitionLLM.ts +++ b/front/lib/api/llm/transitionLLM.ts @@ -221,35 +221,40 @@ export function toBaseMessages( }, ]; case "reasoning": { - if (!c.value.reasoning) { + const reasoning = c.value.reasoning ?? ""; + const { id, encryptedContent } = parseReasoningMetadata( + c.value.metadata + ); + // Empty reasoning summaries can still carry replay state. Some + // reasoners emit such items on tool-use turns, so only discard + // an empty block when it has neither an item id nor encrypted + // content. + if (!reasoning && !id && !encryptedContent) { return []; } - // OpenAI Responses stores a short reasoning item `id` (kept in - // `signature`) separately from the long `encrypted_content`. Every - // other provider stores its thinking signature under - // `encrypted_content`, which we carry directly in `signature`. - if (c.value.provider !== "openai") { + // Responses APIs store a short reasoning item `id` (kept in + // `signature`) separately from the long `encrypted_content`. + // Detect that wire format from the persisted id rather than the + // model provider: Fireworks also exposes the Responses API. + if (id || c.value.provider === "openai") { return [ { role: "assistant", type: "reasoning", - content: { value: c.value.reasoning }, - signature: extractEncryptedContentFromMetadata( - c.value.metadata - ), + content: { value: reasoning }, + signature: id, + ...(encryptedContent ? { encryptedContent } : {}), }, ]; } - const { id, encryptedContent } = parseReasoningMetadata( - c.value.metadata - ); return [ { role: "assistant", type: "reasoning", - content: { value: c.value.reasoning }, - signature: id, - encryptedContent, + content: { value: reasoning }, + signature: extractEncryptedContentFromMetadata( + c.value.metadata + ), }, ]; } @@ -337,10 +342,11 @@ export function withMessageCacheBreakpoints( return result; } -// The new router nests reasoning replay state under `metadata.content`: OpenAI -// uses `id` + `encryptedContent`, Anthropic/Gemini use `signature`. Persist it in -// the legacy top-level shape (`id` / `encrypted_content`) the replay path reads, -// matching what the old router writes. +// The new router nests reasoning replay state under `metadata.content`: +// Responses APIs use `id` + `encryptedContent`, while other APIs use +// `signature`. Persist it in the legacy top-level shape (`id` / +// `encrypted_content`) the replay path reads, matching what the old router +// writes. export function reasoningContentToLegacyMetadata( content: Record | undefined ): { id?: string; encrypted_content?: string } { diff --git a/front/lib/api/llm/utils/index.ts b/front/lib/api/llm/utils/index.ts index b30aff5879b4..35a16da38a43 100644 --- a/front/lib/api/llm/utils/index.ts +++ b/front/lib/api/llm/utils/index.ts @@ -29,8 +29,8 @@ export function extractEncryptedContentFromMetadata(metadata: string): string { return encryptedContent; } -// OpenAI Responses replays a reasoning item by its short `id` and (for ZDR -// regions) its long `encrypted_content`. +// Responses APIs replay a reasoning item by its short `id` and, when returned, +// its long `encrypted_content`. export function parseReasoningMetadata(metadata: string): { id: string; encryptedContent: string; diff --git a/front/lib/model_constructors/providers/fireworks/models/kimi_k3.test.ts b/front/lib/model_constructors/providers/fireworks/models/kimi_k3.test.ts index 9f15d887fdb7..c7a4f186010e 100644 --- a/front/lib/model_constructors/providers/fireworks/models/kimi_k3.test.ts +++ b/front/lib/model_constructors/providers/fireworks/models/kimi_k3.test.ts @@ -37,7 +37,7 @@ describe("Kimi K3 model configuration", () => { ); }); - it("uses Fireworks Priority with its matching token prices", () => { + it("uses Fireworks Responses in priority mode without storage", () => { const endpoint = new MoonshotAiKimiK3GlobalFireworksStream({ FIREWORKS_API_KEY: "test", }); @@ -46,6 +46,8 @@ describe("Kimi K3 model configuration", () => { MoonshotAiKimiK3GlobalFireworksStream.configSchema.parse({}) ); + expect(payload.model).toBe("accounts/fireworks/models/kimi-k3"); + expect(payload.store).toBe(false); expect(payload.service_tier).toBe("priority"); expect(MoonshotAiKimiK3GlobalFireworksStream.tokenPricing).toEqual({ cacheHit: 0.375, diff --git a/front/lib/model_constructors/providers/fireworks/models/kimi_k3.ts b/front/lib/model_constructors/providers/fireworks/models/kimi_k3.ts index 3c628a7889c5..04ce8a8d413a 100644 --- a/front/lib/model_constructors/providers/fireworks/models/kimi_k3.ts +++ b/front/lib/model_constructors/providers/fireworks/models/kimi_k3.ts @@ -18,9 +18,9 @@ const DEFAULT_REASONING_EFFORT = "maximal"; // (https://docs.fireworks.ai/guides/reasoning), so it is not a contradiction — // we follow the model author. Our `maximal` maps to Moonshot's `max`. // -// Confirmed live through Fireworks on 2026-07-27 with the widest -// `inputConfigSchema`: low/high/max all reason. The gateway is looser than -// either doc — it also accepts `medium`, `xhigh` and `none` (which does turn +// Confirmed live through Fireworks' Responses API on 2026-07-28: +// low/high/max all reason. A 2026-07-27 broad gateway characterization also +// found that Fireworks accepts `medium`, `xhigh` and `none` (which does turn // thinking off, despite K3 being described as "thinking permanently enabled"), // and rejects only `minimal`. We expose the documented set only, since // undocumented efforts can change without notice. diff --git a/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.test.ts b/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.test.ts index cf6c94f82be7..6dae573dfaaf 100644 --- a/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.test.ts +++ b/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.test.ts @@ -1,10 +1,13 @@ +import { createAsyncGenerator } from "@app/lib/api/llm/utils"; import * as converters from "@app/lib/model_constructors/sdk/openai_responses/converters/output/utils"; import { outputItemToEvents, + rawOutputToEvents, usageToTokenUsageEvent, } from "@app/lib/model_constructors/sdk/openai_responses/converters/output/utils"; import type { ResponseOutputItem, + ResponseStreamEvent, ResponseUsage, } from "openai/resources/responses/responses"; import { describe, expect, it } from "vitest"; @@ -17,6 +20,26 @@ const metadata = { } as const; describe("outputItemToEvents", () => { + it("preserves an id-bearing reasoning item with no visible summary", () => { + const item = { + type: "reasoning", + id: "rs_empty", + summary: [], + status: "completed", + } satisfies ResponseOutputItem; + + expect(outputItemToEvents(item, metadata, converters)).toEqual([ + { + type: "reasoning", + content: { value: "" }, + metadata: { + ...metadata, + content: { id: "rs_empty" }, + }, + }, + ]); + }); + it("preserves a discovered function call namespace", () => { const item = { type: "function_call", @@ -107,3 +130,92 @@ describe("usageToTokenUsageEvent", () => { }); }); }); + +describe("rawOutputToEvents", () => { + it("preserves interleaved reasoning and function-call item order", async () => { + const rawEvents: ResponseStreamEvent[] = [ + { + type: "response.output_item.done", + output_index: 0, + sequence_number: 0, + item: { + type: "reasoning", + id: "rs_1", + summary: [{ type: "summary_text", text: "first thought" }], + status: "completed", + }, + }, + { + type: "response.output_item.done", + output_index: 1, + sequence_number: 1, + item: { + type: "function_call", + id: "fc_1", + call_id: "call_1", + name: "first_tool", + arguments: "{}", + status: "completed", + }, + }, + { + type: "response.output_item.done", + output_index: 2, + sequence_number: 2, + item: { + type: "reasoning", + id: "rs_2", + summary: [{ type: "summary_text", text: "second thought" }], + status: "completed", + }, + }, + { + type: "response.output_item.done", + output_index: 3, + sequence_number: 3, + item: { + type: "function_call", + id: "fc_2", + call_id: "call_2", + name: "second_tool", + arguments: "{}", + status: "completed", + }, + }, + ]; + const events = []; + for await (const event of rawOutputToEvents( + createAsyncGenerator(rawEvents), + metadata, + converters + )) { + events.push(event); + } + + expect(events.at(-1)).toMatchObject({ + type: "success", + content: { + aggregated: [ + { + type: "reasoning", + content: { value: "first thought" }, + metadata: { content: { id: "rs_1" } }, + }, + { + type: "tool_call", + content: { id: "call_1", name: "first_tool" }, + }, + { + type: "reasoning", + content: { value: "second thought" }, + metadata: { content: { id: "rs_2" } }, + }, + { + type: "tool_call", + content: { id: "call_2", name: "second_tool" }, + }, + ], + }, + }); + }); +}); diff --git a/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.ts b/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.ts index fb72eefea304..86735bc2c772 100644 --- a/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.ts +++ b/front/lib/model_constructors/sdk/openai_responses/converters/output/utils.ts @@ -414,17 +414,17 @@ export function outputItemToEvents( }); case "reasoning": { const text = item.summary.map((summary) => summary.text).join("\n\n"); - // Skip empty reasoning items (no summary emitted, e.g. effort "none"). - return text - ? [ - converters.accumulatedReasoningToReasoningEvent( - metadata, - text, - item.id, - item.encrypted_content ?? undefined - ), - ] - : []; + // Preserve the item even when the provider exposes no visible summary: + // its id/encrypted content is replay state required by some reasoners on + // the next tool-use turn. + return [ + converters.accumulatedReasoningToReasoningEvent( + metadata, + text, + item.id, + item.encrypted_content ?? undefined + ), + ]; } case "function_call": return [ diff --git a/front/lib/model_constructors/stream/clients/fireworks.ts b/front/lib/model_constructors/stream/clients/fireworks.ts index 20fbe4e80524..1b1414777a60 100644 --- a/front/lib/model_constructors/stream/clients/fireworks.ts +++ b/front/lib/model_constructors/stream/clients/fireworks.ts @@ -21,7 +21,7 @@ import type { ChatCompletionToolMessageParam, } from "openai/resources/chat/completions"; -const FIREWORKS_BASE_URL = "https://api.fireworks.ai/inference/v1"; +export const FIREWORKS_BASE_URL = "https://api.fireworks.ai/inference/v1"; // Fireworks model ids are stored bare (e.g. `glm-5p2`) but legacy `ModelIdType` // and the Fireworks API both use the full account-scoped path. diff --git a/front/lib/model_constructors/stream/clients/fireworks_responses.test.ts b/front/lib/model_constructors/stream/clients/fireworks_responses.test.ts new file mode 100644 index 000000000000..47b4a65e5715 --- /dev/null +++ b/front/lib/model_constructors/stream/clients/fireworks_responses.test.ts @@ -0,0 +1,155 @@ +// @vitest-environment node + +import { MoonshotAiKimiK3GlobalFireworksStream } from "@app/lib/model_constructors/stream/endpoints/moonshot_ai_kimi_k3_global_fireworks"; +import { describe, expect, it } from "vitest"; + +const TOOLS = [ + { + name: "calculator", + description: "Calculate an expression", + inputSchema: { + type: "object", + properties: { expression: { type: "string" } }, + required: ["expression"], + additionalProperties: false, + }, + }, + { + name: "get_weather", + description: "Get the weather", + inputSchema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + additionalProperties: false, + }, + }, +]; + +describe("FireworksResponsesStream", () => { + it("replays reasoning and tool calls as Responses input items", () => { + const endpoint = new MoonshotAiKimiK3GlobalFireworksStream({ + FIREWORKS_API_KEY: "test", + }); + const payload = endpoint.buildRequestPayload( + { + conversation: { + system: [ + { + role: "system", + type: "text", + content: { value: "Be concise." }, + }, + ], + messages: [ + { + role: "user", + type: "text", + content: { value: "What is 2 + 3?" }, + }, + { + role: "assistant", + type: "reasoning", + content: { value: "I should use the calculator." }, + signature: "rs_123", + }, + { + role: "assistant", + type: "tool_call_request", + content: { + callId: "call_123", + toolName: "calculator", + arguments: '{"expression":"2 + 3"}', + }, + }, + { + role: "user", + type: "tool_call_result", + content: { + callId: "call_123", + toolName: "calculator", + parts: [{ type: "text", text: "5" }], + isError: false, + }, + }, + ], + }, + }, + MoonshotAiKimiK3GlobalFireworksStream.configSchema.parse({ tools: TOOLS }) + ); + + expect(payload).toMatchObject({ + model: "accounts/fireworks/models/kimi-k3", + service_tier: "priority", + store: false, + input: [ + { + role: "developer", + content: [{ type: "input_text", text: "Be concise." }], + }, + { + role: "user", + content: [{ type: "input_text", text: "What is 2 + 3?" }], + }, + { + id: "rs_123", + type: "reasoning", + summary: [ + { + type: "summary_text", + text: "I should use the calculator.", + }, + ], + }, + { + type: "function_call", + call_id: "call_123", + name: "calculator", + arguments: '{"expression":"2 + 3"}', + }, + { + type: "function_call_output", + call_id: "call_123", + output: "5", + }, + ], + }); + }); + + it("forces a function using the Fireworks-compatible tool choice", () => { + const endpoint = new MoonshotAiKimiK3GlobalFireworksStream({ + FIREWORKS_API_KEY: "test", + }); + const payload = endpoint.buildRequestPayload( + { conversation: { system: [], messages: [] } }, + MoonshotAiKimiK3GlobalFireworksStream.configSchema.parse({ + tools: TOOLS, + forceTool: "calculator", + }) + ); + + expect(payload.tool_choice).toBe("required"); + expect(payload.tools).toEqual([ + expect.objectContaining({ type: "function", name: "calculator" }), + ]); + }); + + it("does not send OpenAI hosted tool search fields to Fireworks", () => { + const endpoint = new MoonshotAiKimiK3GlobalFireworksStream({ + FIREWORKS_API_KEY: "test", + }); + const payload = endpoint.buildRequestPayload( + { conversation: { system: [], messages: [] } }, + MoonshotAiKimiK3GlobalFireworksStream.configSchema.parse({ + tools: TOOLS.map((tool) => ({ ...tool, eager: false })), + toolSearchEnabled: true, + }) + ); + + expect(payload.tools).toHaveLength(2); + expect(payload.tools).toEqual([ + expect.not.objectContaining({ defer_loading: expect.anything() }), + expect.not.objectContaining({ defer_loading: expect.anything() }), + ]); + }); +}); diff --git a/front/lib/model_constructors/stream/clients/fireworks_responses.ts b/front/lib/model_constructors/stream/clients/fireworks_responses.ts new file mode 100644 index 000000000000..3abc2a2a7ef4 --- /dev/null +++ b/front/lib/model_constructors/stream/clients/fireworks_responses.ts @@ -0,0 +1,105 @@ +import { + type FireworksInputConfig, + fireworksConfigSchema, +} from "@app/lib/model_constructors/providers/fireworks/inputConfig"; +import { WithOpenAIResponsesInputConverter } from "@app/lib/model_constructors/sdk/openai_responses/converters/input"; +import { WithOpenAIResponsesOutputConverter } from "@app/lib/model_constructors/sdk/openai_responses/converters/output"; +import { rawOutputToEvents } from "@app/lib/model_constructors/sdk/openai_responses/converters/output/utils"; +import { + FIREWORKS_BASE_URL, + FIREWORKS_MODEL_PREFIX, +} from "@app/lib/model_constructors/stream/clients/fireworks"; +import { StreamEndpoint } from "@app/lib/model_constructors/stream/endpoint"; +import type { Credentials } from "@app/lib/model_constructors/types/credentials"; +import { FIREWORKS_HOST } from "@app/lib/model_constructors/types/hosts"; +import type { Payload } from "@app/lib/model_constructors/types/input/messages"; +import type { ModelResponseEvent } from "@app/lib/model_constructors/types/output/events"; +import OpenAI from "openai"; +import type { + ResponseCreateParamsNonStreaming, + ResponseCreateParamsStreaming, + ResponseStreamEvent, +} from "openai/resources/responses/responses"; + +// Fireworks exposes an OpenAI-compatible Responses API: +// https://docs.fireworks.ai/guides/response-api +export abstract class FireworksResponsesStream extends WithOpenAIResponsesInputConverter( + WithOpenAIResponsesOutputConverter( + StreamEndpoint< + ResponseCreateParamsNonStreaming, + ResponseStreamEvent, + FireworksInputConfig + > + ) +) { + static readonly host = FIREWORKS_HOST; + + static readonly configSchema = fireworksConfigSchema; + + private readonly client: OpenAI; + + constructor({ FIREWORKS_API_KEY }: Credentials) { + super(); + this.client = new OpenAI({ + apiKey: FIREWORKS_API_KEY, + baseURL: FIREWORKS_BASE_URL, + }); + } + + override buildRequestPayload( + payload: Payload, + config: FireworksInputConfig + ): ResponseCreateParamsNonStreaming { + // Fireworks implements Responses function tools, but not OpenAI's hosted + // tool search/deferred-loading extension. + const request = super.buildRequestPayload(payload, { + ...config, + toolSearchEnabled: false, + }); + const forceTool = config.forceTool; + const forcedTools = + forceTool === undefined + ? [] + : (request.tools ?? []).filter( + (tool) => tool.type === "function" && tool.name === forceTool + ); + + return { + ...request, + model: `${FIREWORKS_MODEL_PREFIX}${this.constructor.model}`, + // Dust replays the complete Responses transcript, including reasoning + // item ids and function calls, so provider-side response storage is not + // needed. + store: false, + // Fireworks does not accept OpenAI's named function tool-choice object. + // Restricting the request to that function and requiring a tool call has + // the same forced-tool semantics. + ...(forcedTools.length > 0 + ? { + tools: forcedTools, + tool_choice: "required" as const, + } + : {}), + }; + } + + async *streamRaw( + input: ResponseCreateParamsNonStreaming + ): AsyncGenerator { + const streamingInput: ResponseCreateParamsStreaming = { + ...input, + stream: true, + }; + const stream = await this.client.responses.create(streamingInput); + + for await (const event of stream) { + yield event; + } + } + + async *rawStreamOutputToEvents( + stream: AsyncGenerator + ): AsyncGenerator { + yield* rawOutputToEvents(stream, this.metadata(), this); + } +} diff --git a/front/lib/model_constructors/stream/endpoints/moonshot_ai_kimi_k3_global_fireworks.ts b/front/lib/model_constructors/stream/endpoints/moonshot_ai_kimi_k3_global_fireworks.ts index e22a68f997c6..72f7dfcd1924 100644 --- a/front/lib/model_constructors/stream/endpoints/moonshot_ai_kimi_k3_global_fireworks.ts +++ b/front/lib/model_constructors/stream/endpoints/moonshot_ai_kimi_k3_global_fireworks.ts @@ -1,22 +1,23 @@ import type { FireworksInputConfig } from "@app/lib/model_constructors/providers/fireworks/inputConfig"; import { WithMoonshotAiKimiK3Config } from "@app/lib/model_constructors/providers/fireworks/models/kimi_k3"; -import { FireworksStream } from "@app/lib/model_constructors/stream/clients/fireworks"; +import { FireworksResponsesStream } from "@app/lib/model_constructors/stream/clients/fireworks_responses"; import type { StreamEndpointConstructor } from "@app/lib/model_constructors/stream/configuration"; import type { Payload } from "@app/lib/model_constructors/types/input/messages"; import { MOONSHOT_AI_LAB } from "@app/lib/model_constructors/types/labs"; import { GLOBAL } from "@app/lib/model_constructors/types/regions"; -import type { ChatCompletionCreateParamsStreaming } from "openai/resources/chat/completions"; +import type { ResponseCreateParamsNonStreaming } from "openai/resources/responses/responses"; export class MoonshotAiKimiK3GlobalFireworksStream extends WithMoonshotAiKimiK3Config( - FireworksStream + FireworksResponsesStream ) { - // https://docs.fireworks.ai/serverless/serving-paths override buildRequestPayload( payload: Payload, config: FireworksInputConfig - ): ChatCompletionCreateParamsStreaming { + ): ResponseCreateParamsNonStreaming { return { ...super.buildRequestPayload(payload, config), + // Fireworks accepts this typed Responses extension even though it is not + // yet listed in the Responses API reference. Verified live 2026-07-28. service_tier: "priority", }; }