Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/commands/interaction/output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,23 @@ describe('press CLI output', () => {
);
});

test('prints the response warning after the tap line', () => {
const output = formatPress({
message: 'Tapped (278, 817)',
x: 278,
y: 817,
warning:
'press id="request-mic" opened an Android permission dialog (com.google.android.permissioncontroller) over com.example.app. Use "alert get" to inspect it, then "alert accept" or "alert dismiss" to respond.',
});

expect(output.text).toBe(
[
'Tapped (278, 817)',
'Warning: press id="request-mic" opened an Android permission dialog (com.google.android.permissioncontroller) over com.example.app. Use "alert get" to inspect it, then "alert accept" or "alert dismiss" to respond.',
].join('\n'),
);
});

test('appends the unchanged interactive tail after a removals-only diff', () => {
const output = formatPress({
message: 'Tapped @e4 (100, 200)',
Expand Down
14 changes: 9 additions & 5 deletions src/commands/interaction/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,23 @@ function tapCliOutput(result: CommandRequestResult): CliOutput {
const y = data.y;
if (!ref || typeof x !== 'number' || typeof y !== 'number') {
const output = defaultCommandCliOutput(data);
return { data: output.data, text: appendSettleText(output.text, data.settle) };
return { data: output.data, text: appendResponseNotes(output.text, data) };
}
return { data, text: appendSettleText(`Tapped @${ref} (${x}, ${y})`, data.settle) };
return { data, text: appendResponseNotes(`Tapped @${ref} (${x}, ${y})`, data) };
}

function messageWithSettleCliOutput(result: CommandRequestResult): CliOutput {
const data = result as Record<string, unknown>;
const output = defaultCommandCliOutput(data);
return { data: output.data, text: appendSettleText(output.text, data.settle) };
return { data: output.data, text: appendResponseNotes(output.text, data) };
}

function appendSettleText(text: string | null | undefined, settle: unknown): string {
return `${text ?? ''}${formatSettleText(settle)}`;
function appendResponseNotes(
text: string | null | undefined,
data: Record<string, unknown>,
): string {
const warning = typeof data.warning === 'string' ? `\nWarning: ${data.warning}` : '';
return `${text ?? ''}${warning}${formatSettleText(data.settle)}`;
}

type SettleTextView = {
Expand Down
57 changes: 57 additions & 0 deletions src/daemon/handlers/__tests__/interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1843,6 +1843,63 @@ test('press @ref fails when Android tap escapes to Settings', async () => {
});
});

const ANDROID_PERMISSION_PROMPT_PACKAGES = [
'com.android.permissioncontroller',
'com.google.android.permissioncontroller',
'com.google.android.packageinstaller',
'com.android.packageinstaller',
] as const;

test.each(ANDROID_PERMISSION_PROMPT_PACKAGES)(
'press @ref succeeds with a pending-alert warning when %s foregrounds',
async (packageName) => {
const sessionStore = makeSessionStore();
const sessionName = 'android-permission-prompt';
const session = makeAndroidSession(sessionName);
session.snapshot = {
nodes: attachRefs([
{
index: 0,
type: 'android.widget.Button',
label: 'Request microphone',
rect: { x: 16, y: 40, width: 120, height: 48 },
enabled: true,
hittable: true,
},
]),
createdAt: Date.now(),
backend: 'android',
};
sessionStore.set(sessionName, session);

mockDispatch.mockResolvedValue({ pressed: true });
mockGetAndroidAppState.mockResolvedValue({
package: packageName,
activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity',
});

const response = await handleInteractionCommands({
req: {
token: 't',
session: sessionName,
command: 'press',
positionals: ['@e1'],
flags: {},
},
sessionName,
sessionStore,
contextFromFlags,
});

expect(response?.ok).toBe(true);
if (response?.ok) {
expect(response.data?.warning).toMatch(/opened an Android permission dialog/);
expect(response.data?.warning).toMatch(/"alert get"/);
}
expect(sessionStore.get(sessionName)?.actions).toHaveLength(1);
},
);

test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'verify-press';
Expand Down
23 changes: 17 additions & 6 deletions src/daemon/handlers/interaction-android-escape.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getAndroidAppState } from '../../platforms/android/app-lifecycle.ts';
import { isAndroidPermissionPackage } from '../../platforms/android/alert-detection.ts';
import { AppError } from '@agent-device/kernel/errors';
import type { SessionState } from '../types.ts';

Expand All @@ -9,12 +10,22 @@ export type AndroidEscapeSurface = {
hint: string;
};

/**
* Post-press escape guard. Throws when the tap left the app for a genuine
* escape surface (settings/systemui/launcher). A foregrounded permission
* prompt is NOT an escape — the press succeeded and raised a system dialog
* the agent consumes via `alert` — so it returns a response warning instead.
*/
export async function assertAndroidPressStayedInApp(
session: SessionState,
targetLabel: string,
): Promise<void> {
): Promise<string | undefined> {
const surface = await detectAndroidEscapeSurface(session);
if (!surface) return;
if (!surface) return undefined;

if (isAndroidPermissionPackage(surface.foregroundPackage)) {
return `press ${targetLabel} opened an Android permission dialog (${surface.foregroundPackage}) over ${surface.expectedPackage}. ${surface.hint}`;
}

throw new AppError(
'COMMAND_FAILED',
Expand Down Expand Up @@ -42,7 +53,7 @@ export async function detectAndroidEscapeSurface(
}

export function describeAndroidEscapeSurface(surface: AndroidEscapeSurface): string {
if (surface.foregroundPackage === 'com.google.android.permissioncontroller') {
if (isAndroidPermissionPackage(surface.foregroundPackage)) {
return `Android permission dialog is blocking ${surface.expectedPackage}`;
}
return `${surface.foregroundPackage} is foreground instead of ${surface.expectedPackage}`;
Expand All @@ -57,8 +68,8 @@ export function isAndroidEscapeError(error: AppError): boolean {
}

function buildAndroidEscapeHint(packageName: string): string {
if (packageName === 'com.google.android.permissioncontroller') {
return 'Dismiss or allow the permission prompt, then retry the smoke assertion.';
if (isAndroidPermissionPackage(packageName)) {
return 'Use "alert get" to inspect it, then "alert accept" or "alert dismiss" to respond.';
}
return 'Use screenshot as visual truth, then take a fresh snapshot -i before retrying.';
}
Expand All @@ -67,7 +78,7 @@ function looksLikeAndroidEscapeSurface(packageName: string): boolean {
return (
packageName === 'com.android.settings' ||
packageName === 'com.android.systemui' ||
packageName === 'com.google.android.permissioncontroller' ||
isAndroidPermissionPackage(packageName) ||
packageName.includes('launcher')
);
}
28 changes: 17 additions & 11 deletions src/daemon/handlers/interaction-touch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,8 @@ async function dispatchTargetedTouchViaRuntime(
expectedResolvedTarget: replayTargetGuard,
}),
afterRun: async (result) => {
if (session.lease?.leaseProvider) return;
await assertAndroidPressStayedInApp(
if (session.lease?.leaseProvider) return undefined;
return await assertAndroidPressStayedInApp(
session,
formatTouchTargetLabel(parsedTarget.target, result),
);
Expand Down Expand Up @@ -766,7 +766,8 @@ async function dispatchRuntimeInteraction<
*/
refContext?: RefAdmissionContext;
run(runtime: ReturnType<typeof createInteractionRuntime>): Promise<TResult>;
afterRun?(result: TResult): Promise<void>;
/** May return a warning to append to the successful response (e.g. a pending Android permission dialog). */
afterRun?(result: TResult): Promise<string | undefined>;
buildPayloads(
result: TResult,
): InteractionResponsePayloads | Promise<InteractionResponsePayloads>;
Expand All @@ -777,27 +778,32 @@ async function dispatchRuntimeInteraction<
const runtime = createInteractionRuntime(params);
const actionStartedAt = Date.now();
try {
let afterRunWarning: string | undefined;
const outcome = await runWithAndroidDialogReadinessCheck(
session,
params.req.command,
{ refContext: options.refContext },
async () => {
const result = await options.run(runtime);
await options.afterRun?.(result);
afterRunWarning = await options.afterRun?.(result);
return result;
},
);
if (outcome.aborted) return outcome.response;
const { readiness, runtimeResult } = outcome;
const actionFinishedAt = Date.now();
const { result, responseData, recordedTarget } = await options.buildPayloads(runtimeResult);
if (readiness.status === 'recovered') {
// Append, don't clobber — the builder may already carry a warning
// (e.g. stale-refs, #1076).
const warning =
typeof responseData.warning === 'string'
? `${responseData.warning} ${readiness.warning}`
: readiness.warning;
// Append, don't clobber — the builder may already carry a warning
// (e.g. stale-refs, #1076).
const appendedWarnings = [
...(readiness.status === 'recovered' ? [readiness.warning] : []),
...(afterRunWarning ? [afterRunWarning] : []),
];
if (appendedWarnings.length > 0) {
const warning = [
...(typeof responseData.warning === 'string' ? [responseData.warning] : []),
...appendedWarnings,
].join(' ');
result.warning = warning;
responseData.warning = warning;
}
Expand Down
5 changes: 5 additions & 0 deletions src/platforms/android/alert-detection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ const ANDROID_PERMISSION_PACKAGES = new Set([
'com.google.android.packageinstaller',
'com.android.packageinstaller',
]);

/** Packages that host runtime-permission and install prompts — a legitimate `alert` source, never an app escape. */
export function isAndroidPermissionPackage(packageName: string): boolean {
return ANDROID_PERMISSION_PACKAGES.has(packageName);
}
const ANDROID_SYSTEM_DIALOG_PACKAGES = new Set(['android', 'com.android.systemui']);
const ANDROID_ALERT_ID_PATTERN =
/^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i;
Expand Down
Loading