Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/command-registry/src/flag-definitions-action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
min: 0,
Comment thread
thymikee marked this conversation as resolved.
usageLabel: '--settle-quiet <ms>',
usageDescription:
'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline',
'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline; a replacement from a screen the pre-action capture read as presenting nothing, such as a tab switch, gets no 1500ms confirmation and keeps the asked window',
projectConfig: true,
recorded: false,
},
Expand Down
18 changes: 16 additions & 2 deletions packages/daemon/src/execution/generic-settle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ import type { BoundContextFromFlags } from '../core/context.ts';
import type { RuntimeAdmissionBindings } from '../core/request-runtime-binding.ts';
import type { DaemonPlatformServices } from '../core/platform-services.ts';
import { resolveSettleObservationCapture } from '../core/settle-observation-capture.ts';
import { isOutdatedObservation } from '../core/ref-frame.ts';
import { issueSettleRefs } from './session-snapshot.ts';
import type { SessionStore } from '../core/session-store.ts';
import type { DaemonRequest, DaemonResponse } from '../core/daemon-request.ts';
import type { SessionRef, SessionState } from '../core/session-state.ts';
import type { SnapshotState } from '@agent-device/kernel/snapshot';

/**
* `--settle` on the generic daemon route (#1638): `scroll` and `back` change
Expand Down Expand Up @@ -84,11 +86,23 @@ export function planGenericSettleObservation(
if (invalidSettleFlags) return { response: invalidSettleFlags };
const settle = readSettleRequest(params.flags);
if (!settle) return {};
const snapshot = params.session.snapshot;
const baseline = surfaceScopedNodes(snapshot);
const baseline = storedTreeBaseline(params.session.snapshot);
return { observe: async () => await observeSettled(params, settle, baseline) };
}

/**
* The stored tree as the settle baseline. Its presented-surface fact describes the screen when the
* tree was captured, so a tree a later side effect outdated keeps its nodes for the diff but not
* the fact: an `absent` read taken before a sheet was presented must not rule out the confirmation
* owed to that sheet's dismissal (#3565).
*/
function storedTreeBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes {
const baseline = surfaceScopedNodes(snapshot);
if (!snapshot || !isOutdatedObservation(snapshot)) return baseline;
const { presentedSurface: _outdated, ...current } = baseline;
return current;
}

/**
* Best-effort like the settle engine itself: the action already succeeded, so a
* runtime that cannot even be constructed (session evicted mid-request)
Expand Down
87 changes: 84 additions & 3 deletions src/__tests__/daemon-composition/generic-settle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { composedPlatformServicesFixture } from './platform-services-fixture.ts'
import { beforeEach, expect, test, vi } from 'vitest';
import type { CommandFlags } from '@agent-device/contracts/command';
import type { CommandClock } from '@agent-device/contracts/runtime-contract';
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot';
import {
APP_SURFACE,
iosSystemSurfaceDisclosure,
Expand All @@ -11,7 +11,16 @@ import {
} from '@agent-device/contracts/ios-system-surface';
import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories';
import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory';
import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame';
import {
PRESENTED_SHEET_FULL_NODES,
PRESENTED_SHEET_INTERACTIVE_NODES,
SHEET_DISMISSED_NODES,
} from './presented-sheet-trees.ts';
import {
activateCompleteRefFrame,
markSessionSnapshotOutdated,
refFrameState,
} from '@agent-device/daemon/testing/ref-frame';
import { setSessionSnapshot } from '@agent-device/daemon/testing/session-snapshot';
import type { SessionStore } from '@agent-device/daemon/testing/session-store';
import type { DaemonRequest, DaemonResponse } from '@agent-device/daemon/daemon-request';
Expand Down Expand Up @@ -77,6 +86,7 @@ type SettlePayload = {
captures: number;
quietMs: number;
timeoutMs: number;
waitedMs: number;
refsGeneration?: number;
diff?: {
summary: { additions: number; removals: number; unchanged: number };
Expand Down Expand Up @@ -115,7 +125,7 @@ async function emulateCaptureSnapshotForSession(
}

function mockCommandDispatch(
snapshots: Array<typeof BEFORE_NODES>,
snapshots: RawSnapshotNode[][],
producer: NonNullable<GenericSettleTree['producer']> = 'apple-runner',
) {
let snapshotCalls = 0;
Expand Down Expand Up @@ -181,6 +191,7 @@ type GenericSettleTree = {
nodes: RawSnapshotNode[];
systemSurface?: IosSystemSurfaceProvenance;
producer?: 'apple-runner' | 'simulator-ax-bridge';
presentedSurface?: PresentedSurface;
};

function snapshotPayload(tree: GenericSettleTree) {
Expand All @@ -189,6 +200,7 @@ function snapshotPayload(tree: GenericSettleTree) {
backend: 'xctest' as const,
producer: tree.producer ?? ('apple-runner' as const),
...(tree.systemSurface ? { systemSurface: tree.systemSurface } : {}),
...(tree.presentedSurface ? { presentedSurface: tree.presentedSurface } : {}),
};
}

Expand Down Expand Up @@ -556,6 +568,75 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di
expect(stored.snapshot?.iosSystemSurfaceBundleId).toBeUndefined();
});

/**
* `back --settle` over a sheet left on the session by a capture of one given flavour, which is how
* the generic route picks its baseline (`generic-settle.ts` reads the stored tree directly). The
* settle loop's budget arithmetic runs on the injected clock, so no caller waits real time.
*/
async function dismissSheetFromBaseline(
sessionName: string,
baseline: GenericSettleTree,
options: { outdated?: boolean; captures?: RawSnapshotNode[][] } = {},
): Promise<SettlePayload> {
const sessionStore = makeSessionStore();
const session = seedSession(sessionName, sessionStore, baseline);
if (options.outdated) markSessionSnapshotOutdated(session);
// Rootless, as the first reads after a dismissal can be: only a presented baseline arms on them.
mockCommandDispatch(options.captures ?? [SHEET_DISMISSED_NODES.slice(1)]);
const response = await dispatchGeneric({
sessionName,
sessionStore,
command: 'back',
flags: { settle: true, settleQuietMs: 25, timeoutMs: 30_000 },
clock: fakeClock(),
});
return expectOkData(response).settle as SettlePayload;
}

test('the confirmation arms for one presented sheet whichever baseline flavour the route holds', async () => {
// The interactive capture projects the sheet as 4 nodes and the full read as 6, so a rule keyed on
// the projection disarms on the full one. The producer's fact is stamped before projection.
const interactive = await dismissSheetFromBaseline('generic-settle-dismiss-sheet-interactive', {
nodes: PRESENTED_SHEET_INTERACTIVE_NODES,
presentedSurface: 'presented',
});
const full = await dismissSheetFromBaseline('generic-settle-dismiss-sheet-full', {
nodes: PRESENTED_SHEET_FULL_NODES,
presentedSurface: 'presented',
});

expect(interactive.settled).toBe(true);
expect(full.settled).toBe(true);
expect(interactive.waitedMs).toBeGreaterThanOrEqual(1_500);
expect(full.waitedMs).toBeGreaterThanOrEqual(1_500);
});

test('back --settle keeps the plain window when the stored tree presented nothing', async () => {
// The same broad replacement, stamped `absent` by its producer: the stored tree's fact reaches
// the loop, and a screen with nothing presented over it owes no dismissal confirmation (#3470).
const settle = await dismissSheetFromBaseline(
'generic-settle-absent',
{ nodes: PRESENTED_SHEET_FULL_NODES, presentedSurface: 'absent' },
{ captures: [SHEET_DISMISSED_NODES] },
);

expect(settle.settled).toBe(true);
expect(settle.waitedMs).toBeLessThan(1_500);
});

test('back --settle keeps the confirmation when a side effect outdated an absent stored tree', async () => {
// The app screen was read `absent`, then a press presented the sheet without a capture. The
// stored fact predates the sheet, so it cannot rule out the confirmation its dismissal owes.
const settle = await dismissSheetFromBaseline(
'generic-settle-outdated-absent',
{ nodes: SHEET_DISMISSED_NODES, presentedSurface: 'absent' },
{ outdated: true, captures: [PRESENTED_SHEET_INTERACTIVE_NODES, SHEET_DISMISSED_NODES] },
);

expect(settle.settled).toBe(true);
expect(settle.waitedMs).toBeGreaterThanOrEqual(1_500);
});

test('the settle observation runs after the post-gesture stabilization marker', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'generic-settle-order';
Expand Down
81 changes: 81 additions & 0 deletions src/__tests__/daemon-composition/presented-sheet-trees.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';

/**
* One presented iOS `pageSheet`, in the two flavours a settle baseline can arrive as, plus the app
* screen a dismissal returns to. An interactive capture carries only the surface — the content
* underneath is not hittable, so it drops out of the projection (4 nodes on the iPhone 18 Pro sim,
* October 2026) — while a full read of the very same screen also reports the covered content (6
* nodes). Arming the broad-transition confirmation must not depend on which flavour produced the
* baseline, which is what the caller that reads these asserts.
*/
export const PRESENTED_SHEET_INTERACTIVE_NODES: RawSnapshotNode[] = [
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{
index: 1,
parentIndex: 0,
type: 'StaticText',
label: 'Allow notifications?',
rect: { x: 10, y: 450, width: 370, height: 22 },
},
{
index: 2,
parentIndex: 0,
type: 'Button',
label: 'Continue',
rect: { x: 10, y: 600, width: 370, height: 44 },
hittable: true,
},
{
index: 3,
parentIndex: 0,
type: 'Button',
label: 'Cancel',
rect: { x: 10, y: 660, width: 370, height: 44 },
hittable: true,
},
];

export const PRESENTED_SHEET_FULL_NODES: RawSnapshotNode[] = [
...PRESENTED_SHEET_INTERACTIVE_NODES,
{
index: 4,
parentIndex: 0,
type: 'Other',
label: 'CoveredContent',
rect: { x: 0, y: 0, width: 390, height: 440 },
},
{
index: 5,
parentIndex: 0,
type: 'Button',
label: 'Open settings',
rect: { x: 10, y: 120, width: 370, height: 44 },
},
];

/** The app screen the sheet is dismissed from, which is what every settle capture then reads. */
export const SHEET_DISMISSED_NODES: RawSnapshotNode[] = [
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{
index: 1,
parentIndex: 0,
type: 'StaticText',
label: 'Signed in as demo',
rect: { x: 10, y: 80, width: 200, height: 22 },
},
{
index: 2,
parentIndex: 0,
type: 'Button',
label: 'Home',
rect: { x: 10, y: 120, width: 370, height: 44 },
hittable: true,
},
{
index: 3,
parentIndex: 0,
type: 'TabBar',
label: 'Tab Bar',
rect: { x: 0, y: 800, width: 390, height: 44 },
},
];
84 changes: 84 additions & 0 deletions src/commands/interaction/runtime/settle-presented-surface.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { PresentedSurface, SnapshotState } from '@agent-device/kernel/snapshot';
import { ref, selector } from './selector-read-utils.ts';
import { createSettleDevice } from './__tests__/settle-device-fixtures.ts';
import {
elementCatalogTabSnapshot,
elementSettingsTabSnapshot,
} from './stable-capture.fixtures.ts';

// Each settle route hands the loop its own baseline, and the producer's presented-surface fact has
// to arrive with it (#3565): a tab switch from a screen read `absent` keeps the plain window, while
// the same broad replacement without that reading still pays the confirmation floor. The tab
// switch is the transition #3470 measured; budgets run on the injected clock.

function stamped(snapshot: SnapshotState, presentedSurface: PresentedSurface): SnapshotState {
return { ...snapshot, presentedSurface };
}

/** A press that switches from the catalog tab to the settings tab, with the given baseline fact. */
function tabSwitchDevice(params: { stored: PresentedSurface; preActionCaptureFails?: boolean }) {
const catalog = elementCatalogTabSnapshot();
let captures = 0;
return createSettleDevice({
stored: stamped(catalog, params.stored),
captureSnapshot: () => {
captures += 1;
if (captures === 1 && params.preActionCaptureFails) throw new Error('capture unavailable');
return {
snapshot:
captures === 1
? stamped(catalog, params.stored)
: stamped(elementSettingsTabSnapshot(), 'absent'),
};
},
});
}

function settingsTabRef(): string {
const node = elementCatalogTabSnapshot().nodes.find(
(candidate) => candidate.label === 'Settings',
);
assert.ok(node?.ref);
return `@${node.ref}`;
}

const ROUTES = [
['a selector press', () => selector('label=Settings')],
['an @ref press', () => ref(settingsTabRef())],
] as const;

for (const [label, target] of ROUTES) {
test(`${label} carries an absent baseline into the loop: a tab switch keeps the plain window`, async () => {
const device = tabSwitchDevice({ stored: 'absent' });

const result = await device.interactions.press(target(), { session: 'default', settle: {} });

assert.equal(result.settle?.settled, true);
assert.ok(result.settle.waitedMs < 1_500, `paid the floor: ${result.settle.waitedMs}ms`);
});

test(`${label} keeps the confirmation when the baseline's producer could not answer`, async () => {
const device = tabSwitchDevice({ stored: 'unknown' });

const result = await device.interactions.press(target(), { session: 'default', settle: {} });

assert.equal(result.settle?.settled, true);
assert.ok(result.settle.waitedMs >= 1_500, `lost the floor: ${result.settle.waitedMs}ms`);
});
}

test('a point press whose evidence capture failed does not trust the stored fact', async () => {
// With no capture of its own, the baseline is the stored tree, which may predate a side effect
// this layer cannot see: its `absent` must not rule the confirmation out.
const device = tabSwitchDevice({ stored: 'absent', preActionCaptureFails: true });

const result = await device.interactions.press(
{ kind: 'point', x: 300, y: 820 },
{ session: 'default', settle: {} },
);

assert.equal(result.settle?.settled, true);
assert.ok(result.settle.waitedMs >= 1_500, `trusted a stored fact: ${result.settle.waitedMs}ms`);
});
13 changes: 11 additions & 2 deletions src/commands/interaction/runtime/settle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ async function settleAfterAction(
...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }),
timeoutMs,
resetBudgetOnPrivateAxRecovery: true,
broadTransitionBaselineNodes: params.baseline.nodes,
broadTransitionBaseline: params.baseline,
});
if (outcome.failure) return settleUnavailable(base, outcome.quietMs, outcome.failure.error);
return await readSettledOutcome(runtime, options, params, base, outcome);
Expand Down Expand Up @@ -271,8 +271,17 @@ function authorizedRefBaseline(
return nonEmptyBaseline(surfaceScopedNodes(session?.refFrameSnapshot));
}

/**
* The latest stored tree, for a target no capture resolved. The diff can use its nodes, but its
* presented-surface fact may predate a side effect this layer cannot see, and an `absent` read
* taken before a sheet was presented must not rule out the confirmation owed to that sheet's
* dismissal (#3565).
*/
function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes {
return surfaceScopedNodes(session?.refFrameSnapshot ?? session?.snapshot);
const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes(
session?.refFrameSnapshot ?? session?.snapshot,
);
return baseline;
}

function nonEmptyBaseline(
Expand Down
Loading
Loading