variables for observe

packages/core/examples/observe_variables_login.ts

@@ -0,0 +1,115 @@
+/**
+ * This example shows how to use observe({ variables }) to plan a sensitive
+ * login flow, validate the returned placeholder actions, and then execute them
+ * with act().
+ *
+ * observe() returns %variableName% placeholders in action arguments. That lets
+ * you review the planned actions before any real secret values are used.
+ */
+import { Action, Stagehand } from "../lib/v3/index.js";
+import chalk from "chalk";
+
+const variables = {
+  username: "test@browserbase.com",
+  password: "stagehand=goated",
+};
+
+const loginInstruction = [
+  "Fill the login form using the available variables.",
+  "Use %username% for the email field.",
+  "Use %password% for the password field.",
+  "Include the field name in each action description.",
+].join(" ");
+
+function findValidatedAction(
+  observed: Action[],
+  placeholder: string,
+  keywords: string[],
+): Action {
+  const matches = observed.filter((action) => {
+    const description = action.description.toLowerCase();
+    return (
+      action.arguments?.includes(placeholder) === true &&
+      keywords.some((keyword) => description.includes(keyword))
+    );
+  });
+
+  if (matches.length !== 1) {
+    throw new Error(
+      `Expected exactly one safe action for ${placeholder}, found ${matches.length}`,
+    );
+  }
+
+  return matches[0];
+}
+
+async function observeVariablesLogin() {
+  const stagehand = new Stagehand({
+    env: "BROWSERBASE",
+    verbose: 1,
+  });
+
+  await stagehand.init();
+
+  try {
+    const page = stagehand.context.pages()[0];
+
+    await page.goto("https://v0-modern-login-flow.vercel.app/", {
+      waitUntil: "networkidle",
+      timeoutMs: 30000,
+    });
+
+    const observed = await stagehand.observe(loginInstruction, {
+      variables,
+    });
+
+    console.log(
+      `${chalk.green("Observe:")} Placeholder actions found:\n${observed
+        .map(
+          (action) =>
+            `${chalk.yellow(action.description)} -> ${chalk.blue(action.arguments?.join(", ") || "no arguments")}`,
+        )
+        .join("\n")}`,
+    );
+
+    const emailAction = findValidatedAction(observed, "%username%", ["email"]);
+    const passwordAction = findValidatedAction(observed, "%password%", [
+      "password",
+    ]);
+
+    console.log(
+      `\n${chalk.green("Validated:")} Safe actions to execute:\n${[
+        emailAction,
+        passwordAction,
+      ]
+        .map(
+          (action) =>
+            `${chalk.yellow(action.description)} -> ${chalk.blue(action.arguments?.[0] || "no value")}`,
+        )
+        .join("\n")}`,
+    );
+
+    await stagehand.act(emailAction, { variables });
+    await stagehand.act(passwordAction, { variables });
+
+    const [submitButton] = await stagehand.observe("find the sign in button");
+
+    if (!submitButton) {
+      throw new Error("Could not find the sign in button");
+    }
+
+    await stagehand.act(submitButton);
+    console.log(
+      chalk.green(
+        "\nSubmitted login form. Waiting 10 seconds before closing...",
+      ),
+    );
+    await page.waitForTimeout(10000);
+  } finally {
+    await stagehand.close();
+  }
+}
+
+(async () => {
+  await observeVariablesLogin();
+})();

packages/core/lib/inference.ts

@@ -17,6 +17,7 @@ import type {
   StagehandZodObject,
 } from "./v3/zodCompat.js";
 import { SupportedUnderstudyAction } from "./v3/types/private/handlers.js";
+import type { Variables } from "./v3/types/public/agent.js";
 
 // Re-export for backward compatibility
 export type { LLMParsedResponse, LLMUsage } from "./v3/llm/LLMClient.js";
@@ -245,6 +246,7 @@ export async function observe({
   logger,
   logInferenceToFile = false,
   supportedActions,
+  variables,
 }: {
   instruction: string;
   domElements: string;
@@ -253,6 +255,7 @@ export async function observe({
   logger: (message: LogLine) => void;
   logInferenceToFile?: boolean;
   supportedActions?: string[];
+  variables?: Variables;
 }) {
   const isGPT5 = llmClient.modelName.includes("gpt-5"); // TODO: remove this as we update support for gpt-5 configuration options
 
@@ -297,7 +300,11 @@ export async function observe({
   type ObserveResponse = z.infer<typeof observeSchema>;
 
   const messages: ChatMessage[] = [
-    buildObserveSystemPrompt(userProvidedInstructions, supportedActions),
+    buildObserveSystemPrompt(
+      userProvidedInstructions,
+      supportedActions,
+      variables,
+    ),
     buildObserveUserMessage(instruction, domElements),
   ];
 

packages/core/lib/prompt.ts

@@ -1,5 +1,6 @@
 import { ChatMessage } from "./v3/llm/LLMClient.js";
 import type { Variables } from "./v3/types/public/agent.js";
+import { getVariablePromptEntries } from "./v3/agent/utils/variables.js";
 
 export function buildUserInstructionsString(
   userProvidedInstructions?: string,
@@ -112,10 +113,21 @@ Extracted content: ${JSON.stringify(extractionResponse, null, 2)}`,
 export function buildObserveSystemPrompt(
   userProvidedInstructions?: string,
   supportedActions?: string[],
+  variables?: Variables,
 ): ChatMessage {
   const actionsString = supportedActions?.length
     ? `\n\nSupported actions: ${supportedActions.join(", ")}`
     : "";
+  const variableEntries = getVariablePromptEntries(variables);
+  const variablesString = variableEntries.length
+    ? `\n\nAvailable variables: ${variableEntries
+        .map(({ name, description }) => {
+          return description ? `%${name}% (${description})` : `%${name}%`;
+        })
+        .join(
+          ", ",
+        )}. When an action needs a dynamic or sensitive value, return the matching %variableName% placeholder in the action arguments instead of a literal value`
+    : "";
 
   const observeSystemPrompt = `
 You are helping the user automate the browser by finding elements based on what the user wants to observe in the page.
@@ -125,7 +137,7 @@ You will be given:
 2. a hierarchical accessibility tree showing the semantic structure of the page. The tree is a hybrid of the DOM and the accessibility tree.
 
 Return an array of elements that match the instruction if they exist, otherwise return an empty array.
-When returning elements, include the appropriate method from the supported actions list.${actionsString}. When choosing non-left click actions, provide right or middle as the argument.`;
+When returning elements, include the appropriate method from the supported actions list.${actionsString}${variablesString}. When choosing non-left click actions, provide right or middle as the argument.`;
   const content = observeSystemPrompt.replace(/\s+/g, " ");
 
   return {

packages/core/lib/v3/agent/prompts/agentSystemPrompt.ts

@@ -1,5 +1,6 @@
 import type { AgentToolMode, Variables } from "../../types/public/agent.js";
 import { CAPTCHA_SYSTEM_PROMPT_NOTE } from "../utils/captchaSolver.js";
+import { getVariablePromptEntries } from "../utils/variables.js";
 
 export interface AgentSystemPromptOptions {
   url: string;
@@ -214,17 +215,14 @@ export function buildAgentSystemPrompt(
   const variableToolsNote = isHybridMode
     ? "Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields."
     : "Use %variableName% syntax in the act or fillForm tool's action fields.";
+  const variableEntries = getVariablePromptEntries(variables);
   const variablesSection = hasVariables
     ? `<variables>
     <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>
     <usage>${variableToolsNote}</usage>
     <example>To type a password, use: type %password% into the password field</example>
-    ${Object.entries(variables)
-      .map(([name, v]) => {
-        const description =
-          typeof v === "object" && v !== null && "value" in v
-            ? v.description
-            : undefined;
+    ${variableEntries
+      .map(({ name, description }) => {
         return description
           ? `<variable name="${name}">${description}</variable>`
           : `<variable name="${name}" />`;

packages/core/lib/v3/agent/tools/fillform.ts

@@ -46,8 +46,8 @@ export const fillFormTool = (
           .join(", ")}`;
 
         const observeOptions = executionModel
-          ? { model: executionModel, timeout: toolTimeout }
-          : { timeout: toolTimeout };
+          ? { model: executionModel, variables, timeout: toolTimeout }
+          : { variables, timeout: toolTimeout };
         const observeResults = await v3.observe(instruction, observeOptions);
 
         const completed = [] as unknown[];

packages/core/lib/v3/agent/utils/variables.ts

@@ -22,6 +22,21 @@ export function getVariableDescription(v: VariableValue): string | undefined {
   return undefined;
 }
 
+export interface VariablePromptEntry {
+  name: string;
+  description?: string;
+}
+
+export function getVariablePromptEntries(
+  variables?: Variables,
+): VariablePromptEntry[] {
+  if (!variables) return [];
+  return Object.entries(variables).map(([name, value]) => ({
+    name,
+    description: getVariableDescription(value),
+  }));
+}
+
 /**
  * Substitutes %variableName% tokens in text with resolved variable values.
  * Works with both simple and rich variable formats.

packages/core/lib/v3/handlers/observeHandler.ts

@@ -64,7 +64,7 @@ export class ObserveHandler {
   }
 
   async observe(params: ObserveHandlerParams): Promise<Action[]> {
-    const { instruction, page, timeout, selector, model } = params;
+    const { instruction, page, timeout, selector, model, variables } = params;
 
     const llmClient = this.resolveLlmClient(model);
 
@@ -116,6 +116,7 @@ export class ObserveHandler {
       logger: v3Logger,
       logInferenceToFile: this.logInferenceToFile,
       supportedActions: Object.values(SupportedUnderstudyAction),
+      variables,
     });
 
     const {

packages/core/lib/v3/types/private/handlers.ts

@@ -23,6 +23,7 @@ export interface ExtractHandlerParams<T extends StagehandZodSchema> {
 export interface ObserveHandlerParams {
   instruction?: string;
   model?: ModelConfiguration;
+  variables?: Variables;
   timeout?: number;
   selector?: string;
   page: Page;

packages/core/lib/v3/types/public/api.ts

@@ -10,6 +10,12 @@
  */
 import { z } from "zod/v4";
 import type Browserbase from "@browserbasehq/sdk";
+import { VariablesSchema } from "./variables.js";
+export {
+  VariablePrimitiveSchema,
+  VariableValueSchema,
+  VariablesSchema,
+} from "./variables.js";
 
 // =============================================================================
 // Shared Components
@@ -405,13 +411,17 @@ export const ActOptionsSchema = z
       description:
         "Model configuration object or model name string (e.g., 'openai/gpt-5-nano')",
     }),
-    variables: z
-      .record(z.string(), z.string())
-      .optional()
-      .meta({
-        description: "Variables to substitute in the action instruction",
-        example: { username: "john_doe" },
-      }),
+    variables: VariablesSchema.optional().meta({
+      description:
+        "Variables to substitute in the action instruction. Accepts flat primitives or { value, description? } objects.",
+      example: {
+        username: "john_doe",
+        password: {
+          value: "secret123",
+          description: "The login password",
+        },
+      },
+    }),
     timeout: z.number().optional().meta({
       description: "Timeout in ms for the action",
       example: 30000,
@@ -540,6 +550,17 @@ export const ObserveOptionsSchema = z
       description:
         "Model configuration object or model name string (e.g., 'openai/gpt-5-nano')",
     }),
+    variables: VariablesSchema.optional().meta({
+      description:
+        "Variables whose names are exposed to the model so observe() returns %variableName% placeholders in suggested action arguments instead of literal values. Accepts flat primitives or { value, description? } objects.",
+      example: {
+        username: {
+          value: "john@example.com",
+          description: "The login email",
+        },
+        rememberMe: true,
+      },
+    }),
     timeout: z.number().optional().meta({
       description: "Timeout in ms for the observation",
       example: 30000,

packages/core/lib/v3/types/public/methods.ts

@@ -73,6 +73,7 @@ export const pageTextSchema = z.object({
 
 export interface ObserveOptions {
   model?: ModelConfiguration;
+  variables?: Variables;
   timeout?: number;
   selector?: string;
   page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;

packages/core/lib/v3/types/public/variables.ts

@@ -0,0 +1,24 @@
+import { z } from "zod/v4";
+import type { VariableValue, Variables } from "./agent.js";
+
+type VariablePrimitive = string | number | boolean;
+
+export const VariablePrimitiveSchema: z.ZodType<VariablePrimitive> = z
+  .union([z.string(), z.number(), z.boolean()])
+  .meta({ id: "VariablePrimitive" });
+
+export const VariableValueSchema: z.ZodType<VariableValue> = z
+  .union([
+    VariablePrimitiveSchema,
+    z
+      .object({
+        value: VariablePrimitiveSchema,
+        description: z.string().optional(),
+      })
+      .strict(),
+  ])
+  .meta({ id: "VariableValue" });
+
+export const VariablesSchema: z.ZodType<Variables> = z
+  .record(z.string(), VariableValueSchema)
+  .meta({ id: "Variables" });

packages/core/lib/v3/v3.ts

@@ -1378,6 +1378,7 @@ export class V3 {
       const handlerParams: ObserveHandlerParams = {
         instruction,
         model: options?.model,
+        variables: options?.variables,
         timeout: options?.timeout,
         selector: options?.selector,
         page: page!,
@@ -1400,6 +1401,7 @@ export class V3 {
         "observe",
         {
           instruction,
+          variables: options?.variables,
           timeout: options?.timeout,
         },
         results,