From 6b1bd6aaa0711cb239702d7c09d6e791dc62e385 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:05:24 -0700 Subject: [PATCH 1/8] fix: use caller-provided values in fillForm instead of LLM-hallucinated arguments The fillForm tool was passing only field actions to observe(), causing the LLM to hallucinate placeholder values (e.g. "test@example.com") instead of using the actual values provided by the caller. This broke login forms, 2FA flows, and any workflow where specific values matter. Uses a fillIndex counter to correctly align fill results with field values, handling interleaved non-fill actions. Also uses !== undefined instead of truthiness check so empty string values (for clearing fields) work correctly. Fixes #1789 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../fix-fillform-value-hallucination.md | 5 + packages/core/lib/v3/agent/tools/fillform.ts | 8 + .../unit/fillform-value-override.test.ts | 151 ++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 .changeset/fix-fillform-value-hallucination.md create mode 100644 packages/core/tests/unit/fillform-value-override.test.ts diff --git a/.changeset/fix-fillform-value-hallucination.md b/.changeset/fix-fillform-value-hallucination.md new file mode 100644 index 000000000..6e7b1c794 --- /dev/null +++ b/.changeset/fix-fillform-value-hallucination.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Fix fillForm tool using LLM-hallucinated values instead of caller-provided values diff --git a/packages/core/lib/v3/agent/tools/fillform.ts b/packages/core/lib/v3/agent/tools/fillform.ts index b7e0c9198..12b40d8fb 100644 --- a/packages/core/lib/v3/agent/tools/fillform.ts +++ b/packages/core/lib/v3/agent/tools/fillform.ts @@ -56,7 +56,15 @@ export const fillFormTool = ( const completed = [] as unknown[]; const replayableActions: Action[] = []; + let fillIndex = 0; for (const res of observeResults) { + if (res.method === "fill") { + if (fields[fillIndex]?.value !== undefined) { + res.arguments = [fields[fillIndex].value]; + } + fillIndex++; + } + const actOptions = variables ? { variables, timeout: toolTimeout } : { timeout: toolTimeout }; diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts new file mode 100644 index 000000000..208116489 --- /dev/null +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi } from "vitest"; +import { fillFormTool } from "../../lib/v3/agent/tools/fillform.js"; +import type { V3 } from "../../lib/v3/v3.js"; + +/** + * Reproduces the hallucination bug from #1789: + * fillForm receives { action, value } pairs but only passes `action` to + * observe(). The LLM then hallucinates placeholder values like + * "test@example.com" which get typed into form fields instead of the + * caller-provided values. + */ + +function createMockV3( + observeResults: Array<{ + method: string; + arguments: string[]; + elementId?: string; + description?: string; + }>, +) { + const actCalls: Array<{ + method: string; + arguments: string[]; + }> = []; + + const mock = { + logger: vi.fn(), + recordAgentReplayStep: vi.fn(), + act: vi.fn(async (res: { method: string; arguments: string[] }) => { + actCalls.push({ method: res.method, arguments: [...res.arguments] }); + return { success: true, message: "ok", actionDescription: "done", actions: [] }; + }), + observe: vi.fn(async () => observeResults), + actCalls, + }; + + return mock as unknown as V3 & { actCalls: typeof actCalls }; +} + +const toolCtx = { + toolCallId: "t1", + messages: [] as never[], + abortSignal: new AbortController().signal, +}; + +describe("fillForm value override (#1789)", () => { + it("BUG REPRO: without fix, LLM-hallucinated values reach act()", async () => { + // observe() returns hallucinated placeholder values + const v3 = createMockV3([ + { + method: "fill", + arguments: ["test@example.com"], // hallucinated! + elementId: "0-395", + description: "email field", + }, + { + method: "fill", + arguments: ["password123"], // hallucinated! + elementId: "0-396", + description: "password field", + }, + ]); + + const tool = fillFormTool(v3); + await tool.execute!( + { + fields: [ + { action: "type email into email field", value: "tiwa@trysplendor.com" }, + { action: "type password into password field", value: "actualSecret!" }, + ], + }, + toolCtx, + ); + + // With the fix applied, act() should receive the REAL values + expect(v3.actCalls[0].arguments).toEqual(["tiwa@trysplendor.com"]); + expect(v3.actCalls[1].arguments).toEqual(["actualSecret!"]); + }); + + it("does not override non-fill methods (e.g. click)", async () => { + const v3 = createMockV3([ + { + method: "click", + arguments: [], + elementId: "0-100", + description: "click submit", + }, + ]); + + const tool = fillFormTool(v3); + await tool.execute!( + { + fields: [{ action: "click submit button", value: "ignored" }], + }, + toolCtx, + ); + + // click method should NOT have its arguments overridden + expect(v3.actCalls[0].arguments).toEqual([]); + }); + + it("handles interleaved non-fill actions (fillIndex alignment)", async () => { + // observe() returns a click between two fills + const v3 = createMockV3([ + { method: "click", arguments: [], description: "focus email" }, + { method: "fill", arguments: ["placeholder1"], description: "email" }, + { method: "fill", arguments: ["placeholder2"], description: "password" }, + ]); + + const tool = fillFormTool(v3); + await tool.execute!( + { + fields: [ + { action: "type email", value: "real@email.com" }, + { action: "type password", value: "realPass" }, + ], + }, + toolCtx, + ); + + // The fix should use a fillIndex counter so: + // - click at i=0 is skipped + // - first fill maps to fields[0].value + // - second fill maps to fields[1].value + expect(v3.actCalls[0].arguments).toEqual([]); // click unchanged + expect(v3.actCalls[1].arguments).toEqual(["real@email.com"]); + expect(v3.actCalls[2].arguments).toEqual(["realPass"]); + }); + + it("handles empty string value (clearing a field)", async () => { + const v3 = createMockV3([ + { + method: "fill", + arguments: ["hallucinated"], + description: "search box", + }, + ]); + + const tool = fillFormTool(v3); + await tool.execute!( + { + fields: [{ action: "clear the search box", value: "" }], + }, + toolCtx, + ); + + // Empty string is a valid value (clearing a field) — + // it should NOT fall back to the hallucinated value + expect(v3.actCalls[0].arguments).toEqual([""]); + }); +}); From ca9e6ab811b7a4fb553a1d2df5976fc25dd0d433 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:18:31 -0700 Subject: [PATCH 2/8] fix: skip extra fills from observe and log warning When observe() returns more fill actions than fields provided (e.g. LLM invents a "confirm password" fill), skip the extra fills instead of silently typing hallucinated values. Logs a warning so callers know. Also renames misleading test name per review feedback. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/lib/v3/agent/tools/fillform.ts | 15 ++++++-- .../unit/fillform-value-override.test.ts | 34 ++++++++++++++++++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/packages/core/lib/v3/agent/tools/fillform.ts b/packages/core/lib/v3/agent/tools/fillform.ts index 12b40d8fb..4e221b615 100644 --- a/packages/core/lib/v3/agent/tools/fillform.ts +++ b/packages/core/lib/v3/agent/tools/fillform.ts @@ -59,10 +59,19 @@ export const fillFormTool = ( let fillIndex = 0; for (const res of observeResults) { if (res.method === "fill") { - if (fields[fillIndex]?.value !== undefined) { - res.arguments = [fields[fillIndex].value]; + if (fillIndex < fields.length) { + if (fields[fillIndex].value !== undefined) { + res.arguments = [fields[fillIndex].value]; + } + fillIndex++; + } else { + v3.logger({ + category: "agent", + message: `fillForm: observe returned more fill actions than provided fields (${fields.length}); skipping extra fill`, + level: 1, + }); + continue; } - fillIndex++; } const actOptions = variables diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts index 208116489..6c62f451d 100644 --- a/packages/core/tests/unit/fillform-value-override.test.ts +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -44,7 +44,7 @@ const toolCtx = { }; describe("fillForm value override (#1789)", () => { - it("BUG REPRO: without fix, LLM-hallucinated values reach act()", async () => { + it("uses caller-provided values instead of LLM-hallucinated values", async () => { // observe() returns hallucinated placeholder values const v3 = createMockV3([ { @@ -148,4 +148,36 @@ describe("fillForm value override (#1789)", () => { // it should NOT fall back to the hallucinated value expect(v3.actCalls[0].arguments).toEqual([""]); }); + + it("skips extra fills when observe returns more fills than fields", async () => { + const v3 = createMockV3([ + { method: "fill", arguments: ["hal1"], description: "email" }, + { method: "fill", arguments: ["hal2"], description: "password" }, + { method: "fill", arguments: ["hal3"], description: "confirm password" }, + ]); + + const tool = fillFormTool(v3); + await tool.execute!( + { + fields: [ + { action: "type email", value: "real@email.com" }, + { action: "type password", value: "realPass" }, + ], + }, + toolCtx, + ); + + // Only the two matched fills should be acted on + expect(v3.actCalls).toHaveLength(2); + expect(v3.actCalls[0].arguments).toEqual(["real@email.com"]); + expect(v3.actCalls[1].arguments).toEqual(["realPass"]); + + // Warning should be logged for the skipped fill + expect(v3.logger).toHaveBeenCalledWith( + expect.objectContaining({ + category: "agent", + message: expect.stringContaining("more fill actions than provided fields"), + }), + ); + }); }); From b223c55c799301e9933420e3e1d0c42dd6bf7502 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:24:58 -0700 Subject: [PATCH 3/8] style: format test file with prettier Co-Authored-By: Claude Opus 4.6 (1M context) --- .../unit/fillform-value-override.test.ts | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts index 6c62f451d..5a54ae9c7 100644 --- a/packages/core/tests/unit/fillform-value-override.test.ts +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -28,7 +28,12 @@ function createMockV3( recordAgentReplayStep: vi.fn(), act: vi.fn(async (res: { method: string; arguments: string[] }) => { actCalls.push({ method: res.method, arguments: [...res.arguments] }); - return { success: true, message: "ok", actionDescription: "done", actions: [] }; + return { + success: true, + message: "ok", + actionDescription: "done", + actions: [], + }; }), observe: vi.fn(async () => observeResults), actCalls, @@ -65,8 +70,14 @@ describe("fillForm value override (#1789)", () => { await tool.execute!( { fields: [ - { action: "type email into email field", value: "tiwa@trysplendor.com" }, - { action: "type password into password field", value: "actualSecret!" }, + { + action: "type email into email field", + value: "tiwa@trysplendor.com", + }, + { + action: "type password into password field", + value: "actualSecret!", + }, ], }, toolCtx, @@ -176,7 +187,9 @@ describe("fillForm value override (#1789)", () => { expect(v3.logger).toHaveBeenCalledWith( expect.objectContaining({ category: "agent", - message: expect.stringContaining("more fill actions than provided fields"), + message: expect.stringContaining( + "more fill actions than provided fields", + ), }), ); }); From 8fcaa8edee126e2df594b08ec52235bb26b53d24 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:27:57 -0700 Subject: [PATCH 4/8] style: simplify test file comment to match repo conventions Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/tests/unit/fillform-value-override.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts index 5a54ae9c7..ff0df5125 100644 --- a/packages/core/tests/unit/fillform-value-override.test.ts +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -3,13 +3,8 @@ import { fillFormTool } from "../../lib/v3/agent/tools/fillform.js"; import type { V3 } from "../../lib/v3/v3.js"; /** - * Reproduces the hallucination bug from #1789: - * fillForm receives { action, value } pairs but only passes `action` to - * observe(). The LLM then hallucinates placeholder values like - * "test@example.com" which get typed into form fields instead of the - * caller-provided values. + * Minimal mock of V3 that captures arguments passed to act(). */ - function createMockV3( observeResults: Array<{ method: string; From c36ccd0f72e0c34b2b3a3dcec20e2d9e60c96eb1 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:28:05 -0700 Subject: [PATCH 5/8] style: drop issue ref from test describe block Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/tests/unit/fillform-value-override.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts index ff0df5125..543a0226e 100644 --- a/packages/core/tests/unit/fillform-value-override.test.ts +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -43,7 +43,7 @@ const toolCtx = { abortSignal: new AbortController().signal, }; -describe("fillForm value override (#1789)", () => { +describe("fillForm value override", () => { it("uses caller-provided values instead of LLM-hallucinated values", async () => { // observe() returns hallucinated placeholder values const v3 = createMockV3([ From 61f963b9d7e5c7549768c1f1cad756d6c08e9887 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:28:29 -0700 Subject: [PATCH 6/8] test: replace PII in test fixtures with generic values Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/tests/unit/fillform-value-override.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/tests/unit/fillform-value-override.test.ts b/packages/core/tests/unit/fillform-value-override.test.ts index 543a0226e..b43888b26 100644 --- a/packages/core/tests/unit/fillform-value-override.test.ts +++ b/packages/core/tests/unit/fillform-value-override.test.ts @@ -67,11 +67,11 @@ describe("fillForm value override", () => { fields: [ { action: "type email into email field", - value: "tiwa@trysplendor.com", + value: "user@example.org", }, { action: "type password into password field", - value: "actualSecret!", + value: "s3cret!", }, ], }, @@ -79,8 +79,8 @@ describe("fillForm value override", () => { ); // With the fix applied, act() should receive the REAL values - expect(v3.actCalls[0].arguments).toEqual(["tiwa@trysplendor.com"]); - expect(v3.actCalls[1].arguments).toEqual(["actualSecret!"]); + expect(v3.actCalls[0].arguments).toEqual(["user@example.org"]); + expect(v3.actCalls[1].arguments).toEqual(["s3cret!"]); }); it("does not override non-fill methods (e.g. click)", async () => { From 118fc022610283cff9a5c805c776bbc0e992766f Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:39:36 -0700 Subject: [PATCH 7/8] refactor: remove redundant undefined guard on required field value Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/lib/v3/agent/tools/fillform.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/core/lib/v3/agent/tools/fillform.ts b/packages/core/lib/v3/agent/tools/fillform.ts index 4e221b615..f1b6e08e9 100644 --- a/packages/core/lib/v3/agent/tools/fillform.ts +++ b/packages/core/lib/v3/agent/tools/fillform.ts @@ -60,9 +60,7 @@ export const fillFormTool = ( for (const res of observeResults) { if (res.method === "fill") { if (fillIndex < fields.length) { - if (fields[fillIndex].value !== undefined) { - res.arguments = [fields[fillIndex].value]; - } + res.arguments = [fields[fillIndex].value]; fillIndex++; } else { v3.logger({ From 86eba368308c9f85c85286c5a53175e3f5b3084b Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Tue, 10 Mar 2026 17:46:51 -0700 Subject: [PATCH 8/8] fix: warn when observe returns fewer fills than provided fields Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/lib/v3/agent/tools/fillform.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/core/lib/v3/agent/tools/fillform.ts b/packages/core/lib/v3/agent/tools/fillform.ts index f1b6e08e9..a81bc5ac9 100644 --- a/packages/core/lib/v3/agent/tools/fillform.ts +++ b/packages/core/lib/v3/agent/tools/fillform.ts @@ -81,6 +81,13 @@ export const fillFormTool = ( replayableActions.push(...(actResult.actions as Action[])); } } + if (fillIndex < fields.length) { + v3.logger({ + category: "agent", + message: `fillForm: observe returned fewer fill actions (${fillIndex}) than provided fields (${fields.length}); ${fields.length - fillIndex} field value(s) were not applied`, + level: 1, + }); + } v3.recordAgentReplayStep({ type: "fillForm", fields,