diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts index e9ab3ab95..899ff2254 100644 --- a/src/commands/interaction/runtime/settle.test.ts +++ b/src/commands/interaction/runtime/settle.test.ts @@ -313,51 +313,355 @@ test('the settled diff line list is bounded with a truncation marker', async () assert.equal(diff.truncated, true); }); -test('keyboard Key nodes never spend the settled diff budget', async () => { +// Keyboard chrome fixtures mirror the LIVE capture shape (iPhone 17 Pro sim, +// iOS 26, July 2026, React Navigation playground): the keyboard renders in +// its own dedicated window; the [Keyboard] container (keys + shift/Emoji/ +// return buttons) and the "Next keyboard"/"Dictate" candidate-bar buttons are +// SIBLING subtrees under that window — the candidate-bar buttons are NOT +// descendants of the container, which is exactly what an earlier hand-built +// fixture got wrong (a container-descendant walk alone misses them). +function keyboardWindowNodes(startIndex: number, parentIndex?: number) { + const kb = startIndex; + return [ + { + index: kb, + depth: 1, + ...(parentIndex !== undefined ? { parentIndex } : {}), + type: 'Window', + label: 'Next keyboard', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: kb + 1, + depth: 3, + parentIndex: kb, + type: 'Other', + label: 'Next keyboard', + rect: { x: 0, y: 566, width: 402, height: 308 }, + hittable: true, + }, + { + index: kb + 2, + depth: 4, + parentIndex: kb + 1, + type: 'Other', + label: 'Padding-Left', + rect: { x: 0, y: 583, width: 402, height: 233 }, + }, + { + index: kb + 3, + depth: 5, + parentIndex: kb + 2, + type: 'Keyboard', + label: 'Padding-Left', + rect: { x: 0, y: 583, width: 402, height: 233 }, + }, + ...Array.from({ length: 26 }, (_, key) => ({ + index: kb + 4 + key, + depth: 7, + parentIndex: kb + 3, + type: 'Key', + label: String.fromCharCode(97 + key), + rect: { x: (key % 10) * 40, y: 590 + Math.floor(key / 10) * 54, width: 39, height: 54 }, + })), + ...['shift', 'Emoji', 'return'].map((label, extra) => ({ + index: kb + 30 + extra, + depth: 7, + parentIndex: kb + 3, + type: 'Button', + label, + rect: { x: 5 + extra * 50, y: 752, width: 49, height: 54 }, + })), + // Candidate-bar chrome: siblings of the [Keyboard] container's wrapper, + // not descendants of the container itself. + { + index: kb + 33, + depth: 5, + parentIndex: kb + 1, + type: 'Button', + label: 'Next keyboard', + rect: { x: 8, y: 806, width: 68, height: 69 }, + hittable: true, + }, + { + index: kb + 34, + depth: 5, + parentIndex: kb + 1, + type: 'Button', + label: 'Dictate', + rect: { x: 325, y: 805, width: 68, height: 69 }, + }, + ]; +} + +test('keyboard keys, chrome buttons, and candidate-bar siblings never spend the settled diff budget', async () => { const before = buttonSnapshot(); - // A fill-style settled tree: a summoned keyboard (container + keys) plus the - // content change the agent actually cares about. + // A fill-style settled tree: the summoned keyboard window plus the content + // change the agent actually cares about. const keyboardTree = makeSnapshotState([ { index: 0, depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + ...keyboardWindowNodes(1, 0), + { + index: 36, + depth: 1, + parentIndex: 0, type: 'StaticText', label: 'Results for alpenglow', rect: { x: 0, y: 0, width: 200, height: 20 }, hittable: true, }, + ...['Next', 'Back', 'Home'].map((label, extra) => ({ + index: extra + 37, + depth: 1, + parentIndex: 0, + type: 'Button', + label, + rect: { x: 0, y: 240 + extra * 40, width: 100, height: 40 }, + hittable: true, + })), + ]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : keyboardTree }; + }, + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: {}, + }); + + const diff = result.settle?.diff; + assert.ok(diff); + const texts = diff.lines.map((line) => line.text).join('\n'); + assert.match(texts, /Results for alpenglow/); + assert.match(texts, /\[keyboard\]/); + // The container line survives as the keyboard signal; keys, chrome buttons + // under the container, the candidate-bar siblings (Next keyboard/Dictate), + // and the keyboard window's own wrappers all collapse. + assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text))); + assert.ok(!diff.lines.some((line) => /shift|Emoji|return|Dictate|Next keyboard/.test(line.text))); + // application + keyboard container + StaticText + 3 screen buttons. + assert.equal(diff.summary.additions, 6); +}); + +test('keyboard-only changes (window + container + chrome) do not suppress the settle tail trigger', async () => { + // The field and its screen buttons are unchanged by the fill itself; the + // keyboard summoning is the only tree change. Before #1167's fix, the + // chrome buttons' fresh added-line refs (or, after the diff-only fix, the + // container's own added ref) would read as "the diff already handed back a + // target" and suppress the tail — leaving the still-relevant screen + // buttons invisible. + const controls = [ + { + index: 0, + depth: 0, + type: 'TextField', + label: 'Search', + rect: { x: 0, y: 0, width: 200, height: 40 }, + hittable: true, + }, { index: 1, depth: 0, - type: 'Keyboard', - label: 'keyboard', - rect: { x: 0, y: 500, width: 400, height: 300 }, + type: 'Button', + label: 'Cancel', + rect: { x: 0, y: 50, width: 100, height: 40 }, hittable: true, }, - ...Array.from({ length: 26 }, (_, key) => ({ - index: key + 2, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Search now', + rect: { x: 110, y: 50, width: 100, height: 40 }, + hittable: true, + }, + ]; + const before = makeSnapshotState(controls); + const settledTree = makeSnapshotState([...controls, ...keyboardWindowNodes(3)]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : settledTree }; + }, + }); + + const result = await device.interactions.fill(selector('label=Search'), 'hello', { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + // Only the keyboard container line is added; the window, its wrappers, the + // chrome buttons, and the candidate-bar siblings all collapse. + assert.equal(settle.diff?.summary.additions, 1); + assert.ok( + !settle.diff?.lines.some((line) => /shift|return|Dictate|Next keyboard/i.test(line.text)), + ); + assert.ok(settle.tail, 'keyboard-only additions must not suppress the tail'); + assert.deepEqual( + settle.tail?.map((entry) => entry.label), + ['Search', 'Cancel', 'Search now'], + ); +}); + +test('a filled field relabeling itself (self-echo added refs) does not suppress the settle tail', async () => { + // Live-verified shape: filling a field re-labels the field line with its + // new value (and ancestor wrappers inherit it), so the diff carries added + // refs even though nothing NEW appeared on screen. Those self-echo refs + // (settled node rect contains the action point) must not suppress the + // tail — the next targets after a fill are the screen's unchanged controls. + const before = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'TextField', + rect: { x: 12, y: 129, width: 377, height: 41 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Submit', + rect: { x: 12, y: 182, width: 378, height: 40 }, + hittable: true, + }, + ]); + const settledTree = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'TextField', + label: 'hello', + value: 'hello', + rect: { x: 12, y: 129, width: 377, height: 41 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Submit', + rect: { x: 12, y: 182, width: 378, height: 40 }, + hittable: true, + }, + ]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : settledTree }; + }, + }); + + const result = await device.interactions.fill(ref('@e1'), 'hello', { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + // The relabeled field IS an added line (with a ref) in the diff... + const added = settle.diff?.lines.filter((line) => line.kind === 'added') ?? []; + assert.equal(added.length, 1); + assert.match(added[0]?.text ?? '', /hello/); + assert.ok(added[0]?.ref); + // ...but it re-describes the acted-on element, so the tail still fires + // with the actual next target. + assert.deepEqual(settle.tail, [{ ref: 'e2', role: 'button', label: 'Submit' }]); +}); + +test('a keyboard window hosting an app composer field is never window-classified as chrome', async () => { + // iOS hosts inputAccessoryView content (e.g. a messaging composer) in the + // keyboard window. The conservative guard keeps such windows out of the + // whole-window chrome rule: the composer field/send button must stay + // visible even though candidate-bar chrome then leaks through. + const before = buttonSnapshot(); + const settledTree = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Chat', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, depth: 1, + parentIndex: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + // inputAccessoryView composer: an editable field OUTSIDE the [Keyboard] + // container but INSIDE the keyboard window. + { + index: 2, + depth: 2, parentIndex: 1, - type: 'Key', - label: String.fromCharCode(97 + key), - rect: { x: key * 10, y: 520, width: 10, height: 40 }, + type: 'TextField', + label: 'Message', + rect: { x: 0, y: 500, width: 320, height: 44 }, hittable: true, - })), - ...['Next', 'Back', 'Home'].map((label, extra) => ({ - index: extra + 28, - depth: 0, + }, + { + index: 3, + depth: 2, + parentIndex: 1, type: 'Button', - label, - rect: { x: 0, y: 40 + extra * 40, width: 100, height: 40 }, + label: 'Send', + rect: { x: 330, y: 500, width: 60, height: 44 }, hittable: true, - })), + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'Keyboard', + label: 'keyboard', + rect: { x: 0, y: 566, width: 402, height: 308 }, + }, + { + index: 5, + depth: 3, + parentIndex: 4, + type: 'Key', + label: 'q', + rect: { x: 5, y: 590, width: 39, height: 54 }, + }, + { + index: 6, + depth: 3, + parentIndex: 4, + type: 'Button', + label: 'shift', + rect: { x: 5, y: 698, width: 51, height: 54 }, + }, ]); let captures = 0; const device = createSettleDevice({ stored: before, captureSnapshot: () => { captures += 1; - return { snapshot: captures === 1 ? before : keyboardTree }; + return { snapshot: captures === 1 ? before : settledTree }; }, }); @@ -369,11 +673,11 @@ test('keyboard Key nodes never spend the settled diff budget', async () => { const diff = result.settle?.diff; assert.ok(diff); const texts = diff.lines.map((line) => line.text).join('\n'); - assert.match(texts, /Results for alpenglow/); - assert.match(texts, /keyboard/); - // The container line survives as the keyboard signal; individual keys do not. - assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text))); - assert.equal(diff.summary.additions, 5); + // Composer content survives; container-descendant chrome still collapses. + assert.match(texts, /Message/); + assert.match(texts, /Send/); + assert.match(texts, /\[keyboard\]/); + assert.ok(!diff.lines.some((line) => /\[key\]|shift/.test(line.text))); }); test('added lines win diff-budget slots over removals under truncation', async () => { @@ -514,13 +818,19 @@ test('a removals-only settled diff (modal dismiss) attaches an unchanged interac assert.ok(settle); assert.deepEqual(settle.diff?.summary, { additions: 0, removals: 3, unchanged: 2 }); assert.ok(!settle.diff?.lines.some((line) => line.kind === 'added')); - // The StaticText survives but is not interactive; only the hittable button - // makes the tail. - assert.deepEqual(settle.tail, [{ ref: 'e1', role: 'button', label: 'Add to cart' }]); + // The tail matches `snapshot -i`'s own bar for the settled tree: both + // surviving elements are candidates, `hittable` is not required (#1167 post- + // merge benchmark: real buttons commonly report `hittable: false`/undefined + // right after a dismiss animation, so requiring it silently dropped exactly + // the elements the tail exists to surface). + assert.deepEqual(settle.tail, [ + { ref: 'e1', role: 'button', label: 'Add to cart' }, + { ref: 'e2', role: 'text', label: 'Price: $12' }, + ]); assert.equal(settle.tailTruncated, undefined); }); -test('the unchanged interactive tail excludes non-hittable and covered candidates', async () => { +test('the unchanged interactive tail includes non-hittable candidates but excludes covered ones', async () => { // Every surviving node's attributes (hittable/blocked-relevant fields) must // match a `before` line exactly, or it would read as an added line and // suppress the tail entirely (see the trigger-condition test above). @@ -594,9 +904,73 @@ test('the unchanged interactive tail excludes non-hittable and covered candidate const settle = result.settle; assert.ok(settle); assert.equal(settle.diff?.summary.additions, 0); - // Not hittable, then covered, are both excluded; only the plain hittable - // button remains. - assert.deepEqual(settle.tail, [{ ref: 'e3', role: 'button', label: 'Share' }]); + // `hittable: false` no longer excludes a candidate (matches `snapshot -i`'s + // own bar); `interactionBlocked: 'covered'` still does. + assert.deepEqual(settle.tail, [ + { ref: 'e1', role: 'button', label: 'Add to cart' }, + { ref: 'e3', role: 'button', label: 'Share' }, + ]); +}); + +test('the unchanged interactive tail excludes structural application/window chrome', async () => { + // The flagship #1167 post-merge benchmark case: after a dialog dismiss, the + // old `hittable === true` bar let a lone application/window pair through + // (both routinely report `hittable: true` on their full-screen root frame) + // while excluding the real button because its `hittable` state was + // undefined right after the dismiss animation. + const before = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', label: 'React Navigation Example' }, + { index: 1, depth: 0, type: 'Window' }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Discard and go back', + rect: { x: 10, y: 20, width: 200, height: 40 }, + }, + { + index: 3, + depth: 0, + type: 'Button', + label: 'Cancel', + rect: { x: 10, y: 80, width: 200, height: 40 }, + }, + ]); + // The dialog's own Cancel button is dismissed with it; the application, + // window, and the real actionable button survive unchanged. None of the + // three carry `hittable: true` here, mirroring the real post-dismiss + // capture shape. + const after = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', label: 'React Navigation Example' }, + { index: 1, depth: 0, type: 'Window' }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Discard and go back', + rect: { x: 10, y: 20, width: 200, height: 40 }, + }, + ]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.press(selector('label=Cancel'), { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.diff?.summary.additions, 0); + // Application/window chrome is dropped; the real button is the only entry, + // despite carrying no `hittable: true` flag. + assert.deepEqual(settle.tail, [{ ref: 'e3', role: 'button', label: 'Discard and go back' }]); }); test('the unchanged interactive tail is capped with a truncation marker', async () => { @@ -657,3 +1031,43 @@ test('buildSettleTailEntries dedups candidates already carrying an excluded ref' assert.deepEqual(result.tail, [{ ref: 'e2', role: 'button', label: 'Share' }]); }); + +test('buildSettleTailEntries drops application/window chrome and does not require hittable', () => { + const settledNodes = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', label: 'Example' }, + { index: 1, depth: 0, type: 'Window' }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Discard and go back', + rect: { x: 10, y: 20, width: 100, height: 40 }, + // Deliberately no `hittable` field, mirroring a real post-dismiss + // capture: the tail bar no longer requires `hittable === true`. + }, + ]).nodes; + + const result = buildSettleTailEntries(settledNodes, new Set()); + + assert.deepEqual(result.tail, [{ ref: 'e3', role: 'button', label: 'Discard and go back' }]); +}); + +test('buildSettleTailEntries drops the keyboard container and its chrome descendants', () => { + const settledNodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Send', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + { index: 1, depth: 0, type: 'Keyboard', label: 'keyboard' }, + { index: 2, depth: 1, parentIndex: 1, type: 'Key', label: 'q' }, + { index: 3, depth: 1, parentIndex: 1, type: 'Button', label: 'shift' }, + ]).nodes; + + const result = buildSettleTailEntries(settledNodes, new Set()); + + assert.deepEqual(result.tail, [{ ref: 'e1', role: 'button', label: 'Send' }]); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 06a6e7ed5..fb7381191 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -1,8 +1,9 @@ -import type { SnapshotNode } from '../../../kernel/snapshot.ts'; +import type { Point, SnapshotNode } from '../../../kernel/snapshot.ts'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; import { isSparseSnapshotQualityVerdict } from '../../../snapshot/snapshot-quality.ts'; import { buildSnapshotDiff } from '../../../snapshot/snapshot-diff.ts'; import { displayLabel, formatRole } from '../../../snapshot/snapshot-lines.ts'; +import { normalizeType } from '../../../utils/text-surface.ts'; import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; import type { InteractionEvidence, @@ -94,7 +95,11 @@ export async function settleAfterInteraction( // observation, so surfacing refs would invite agents to act on // advisory state. ...(outcome.settled && stored - ? buildSettleDiffAndTail(resolveBaselineNodes(params.resolved), settledNodes) + ? buildSettleDiffAndTail( + resolveBaselineNodes(params.resolved), + settledNodes, + params.resolved.point, + ) : {}), ...resolveSettleHint(outcome, stored, settledNodes.length), }, @@ -142,8 +147,8 @@ function buildSettleDiff( // snapshot), extra baseline-only lines surface as removals — advisory noise, // the same baseline caveat --verify's changedFromBefore already accepts. const diff = buildSnapshotDiff( - withoutKeyboardKeys(baselineNodes), - withoutKeyboardKeys(settledNodes), + withoutKeyboardChrome(baselineNodes), + withoutKeyboardChrome(settledNodes), { flatten: true, withRefs: true }, ); const changed = diff.lines.filter((line) => line.kind !== 'unchanged'); @@ -162,45 +167,96 @@ function buildSettleDiff( function buildSettleDiffAndTail( baselineNodes: SnapshotNode[], settledNodes: SnapshotNode[], + actionPoint: Point | undefined, ): Pick { const diff = buildSettleDiff(baselineNodes, settledNodes); - return { diff, ...buildSettleTail(diff, settledNodes) }; + return { diff, ...buildSettleTail(diff, settledNodes, actionPoint) }; } /** * Unchanged interactive refs tail: attached ONLY when the settled diff carries - * zero added-line refs (a modal-dismiss/toast-only diff shows removals but - * nothing added, so the next actionable target is otherwise invisible). Every - * hittable, uncovered element on the settled tree is a candidate; refs already - * present on the diff's added lines are excluded so the tail never repeats - * what the diff already handed the caller. + * zero MEANINGFUL added-line refs (a modal-dismiss/toast-only diff shows + * removals but nothing added, so the next actionable target is otherwise + * invisible). "Meaningful" excludes two ref classes that are provably not a + * NEW target (live-verified on the July 2026 React Navigation benchmark flow): + * + * - Keyboard chrome refs: a fill that summons the keyboard adds the keyboard + * container line, but that is not a next target the caller asked for. + * - Self-echo refs: added lines whose settled node's rect contains the action + * point are re-descriptions of the element the caller JUST acted on (a + * filled field re-labels itself with its new value, and its ancestor + * wrappers inherit that label), not something new to press next. + * + * Refs already present on the diff's added lines (meaningful or not) are also + * excluded from the tail itself so it never repeats what the diff already + * handed the caller. */ function buildSettleTail( diff: NonNullable, settledNodes: SnapshotNode[], + actionPoint: Point | undefined, ): Pick { const addedRefs = new Set( diff.lines.filter((line) => line.kind === 'added' && line.ref).map((line) => line.ref), ); - if (addedRefs.size > 0) return {}; + const keyboardChromeRefs = collectKeyboardChromeRefs(settledNodes); + const byRef = new Map(settledNodes.filter((node) => node.ref).map((node) => [node.ref, node])); + const hasMeaningfulAddedRef = [...addedRefs].some( + (ref) => + ref !== undefined && + !keyboardChromeRefs.has(ref) && + !isSelfEchoNode(byRef.get(ref), actionPoint), + ); + if (hasMeaningfulAddedRef) return {}; return buildSettleTailEntries(settledNodes, addedRefs); } +function isSelfEchoNode(node: SnapshotNode | undefined, actionPoint: Point | undefined): boolean { + if (!node?.rect || !actionPoint) return false; + const { x, y, width, height } = node.rect; + return ( + actionPoint.x >= x && + actionPoint.x <= x + width && + actionPoint.y >= y && + actionPoint.y <= y + height + ); +} + +// Structural container roles that survive an interactive-only capture (as a +// lone root or alongside real content) but are never a next actionable +// target: application/window chrome, not a control. `snapshot -i` itself +// still lists these lines, but the tail exists specifically to name pressable +// targets, so it drops them rather than spend budget on chrome. +const STRUCTURAL_TAIL_ROLES = new Set(['application', 'window']); + /** * The filtering/cap step behind `buildSettleTail`, split out so the dedup * rule (excludeRefs) is unit-testable independent of the trigger condition * above. + * + * Inclusion bar matches what `snapshot -i` itself would show for the same + * interactive-only capture: presence in `settledNodes` (already filtered to + * interactive content upstream) IS the interactivity bar, so this does NOT + * additionally require `hittable === true`. A real dismiss-animation capture + * routinely reports transient buttons as `hittable: false`/`undefined` right + * after the dismissing element leaves — requiring `hittable === true` here + * was stricter than `snapshot -i`'s own bar and silently dropped exactly the + * buttons the tail exists to surface (#1167 post-merge benchmark). Structural + * application/window chrome and any keyboard container/chrome subtree are + * excluded on top of that bar: never a next actionable target either way. */ export function buildSettleTailEntries( settledNodes: SnapshotNode[], excludeRefs: ReadonlySet, ): Pick { + const keyboardChromeRefs = collectKeyboardChromeRefs(settledNodes); const candidates = settledNodes.filter( (node) => node.ref && - node.hittable === true && node.interactionBlocked !== 'covered' && - !excludeRefs.has(node.ref), + !excludeRefs.has(node.ref) && + !keyboardChromeRefs.has(node.ref) && + !STRUCTURAL_TAIL_ROLES.has(formatRole(node.type ?? 'Element')), ); if (candidates.length === 0) return {}; const tail: SettleTailEntry[] = candidates.slice(0, MAX_SETTLE_TAIL_ENTRIES).map((node) => { @@ -214,12 +270,148 @@ export function buildSettleTailEntries( }; } -// The iOS QWERTY keyboard is ~50 Key nodes; a fill that summons it would spend -// most of the capped line budget spelling out the keyboard instead of the -// content change the agent actually asked to observe. The Keyboard container -// node stays, so "keyboard appeared/left" remains one visible diff line. -function withoutKeyboardKeys(nodes: SnapshotNode[]): SnapshotNode[] { - return nodes.filter((node) => node.type !== 'Key'); +// Editable text roles (normalized `node.type` vocabulary) used by the +// keyboard-window guard below. +const EDITABLE_TEXT_TYPES = new Set([ + 'textfield', + 'securetextfield', + 'textview', + 'textarea', + 'searchfield', + 'edittext', +]); + +/** + * Keyboard chrome classification, live-verified against the iPhone 17 Pro + * simulator (iOS 26, July 2026): the software keyboard renders in its OWN + * dedicated window (UIRemoteKeyboardWindow). That window contains the + * `[Keyboard]` container (keys plus real XCUIElementTypeButton chrome like + * shift/Emoji/return) AND a SIBLING subtree holding the "Next keyboard" and + * "Dictate" buttons — siblings of the container, so a container-descendant + * walk alone provably misses them. The rule is therefore: every node inside + * a window that has a `[Keyboard]` descendant is keyboard chrome. + * + * Detection stays structural (`parentIndex` chains), never label-based: + * keyboard chrome text is locale-dependent (the verified capture had Polish + * key labels) and a label list would silently stop matching under a + * different input language. + * + * Conservative guard: a window that also hosts an editable text node OUTSIDE + * the keyboard container is never window-classified — iOS hosts + * inputAccessoryView content (e.g. a messaging composer) in the keyboard + * window, and hiding the field the user is typing into would be worse than + * leaking chrome. Such windows fall back to the container-descendant walk. + */ +type KeyboardChrome = { + /** Chrome node indexes EXCLUDING the containers: stripped from the diff. */ + strippedIndexes: ReadonlySet; + /** Refs of ALL chrome incl. containers/window: trigger + tail exclusion. */ + refs: ReadonlySet; +}; + +const EMPTY_KEYBOARD_CHROME: KeyboardChrome = { strippedIndexes: new Set(), refs: new Set() }; + +function collectKeyboardChrome(nodes: SnapshotNode[]): KeyboardChrome { + const containerIndexes = new Set( + nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index), + ); + if (containerIndexes.size === 0) return EMPTY_KEYBOARD_CHROME; + const byIndex = new Map(nodes.map((node) => [node.index, node])); + const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes); + const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, chromeIndexes); + for (const index of collectSubtreeIndexes(nodes, byIndex, windowIndexes)) { + chromeIndexes.add(index); + } + const refs = new Set( + nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), + ); + // The [Keyboard] container line itself survives the diff so "keyboard + // appeared/left" stays one visible signal line; everything else collapses. + const strippedIndexes = new Set( + [...chromeIndexes].filter((index) => !containerIndexes.has(index)), + ); + return { strippedIndexes, refs }; +} + +/** The root indexes plus every node whose parent chain passes through one. */ +function collectSubtreeIndexes( + nodes: SnapshotNode[], + byIndex: Map, + rootIndexes: ReadonlySet, +): Set { + const indexes = new Set(rootIndexes); + if (rootIndexes.size === 0) return indexes; + for (const node of nodes) { + if (hasAncestorIn(node, byIndex, rootIndexes)) indexes.add(node.index); + } + return indexes; +} + +function withoutKeyboardChrome(nodes: SnapshotNode[]): SnapshotNode[] { + const { strippedIndexes } = collectKeyboardChrome(nodes); + if (strippedIndexes.size === 0) return nodes; + return nodes.filter((node) => !strippedIndexes.has(node.index)); +} + +function collectKeyboardChromeRefs(nodes: SnapshotNode[]): ReadonlySet { + return collectKeyboardChrome(nodes).refs; +} + +/** + * Windows eligible for whole-window chrome classification: nearest `[window]` + * ancestor of each `[Keyboard]` container, minus windows hosting editable + * text outside the container subtree (the inputAccessoryView guard above). + */ +function resolveKeyboardWindowIndexes( + nodes: SnapshotNode[], + byIndex: Map, + containerChromeIndexes: ReadonlySet, +): Set { + const windowIndexes = new Set(); + for (const index of containerChromeIndexes) { + const node = byIndex.get(index); + if (!node || normalizeType(node.type ?? '') !== 'keyboard') continue; + const windowAncestor = findNearestWindowAncestor(node, byIndex); + if (windowAncestor !== undefined) windowIndexes.add(windowAncestor); + } + for (const windowIndex of windowIndexes) { + const windowSet = new Set([windowIndex]); + const hostsEditableText = nodes.some( + (node) => + EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) && + !containerChromeIndexes.has(node.index) && + hasAncestorIn(node, byIndex, windowSet), + ); + if (hostsEditableText) windowIndexes.delete(windowIndex); + } + return windowIndexes; +} + +function findNearestWindowAncestor( + node: SnapshotNode, + byIndex: Map, +): number | undefined { + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + while (current) { + if (normalizeType(current.type ?? '') === 'window') return current.index; + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return undefined; +} + +function hasAncestorIn( + node: SnapshotNode, + byIndex: Map, + ancestorIndexes: ReadonlySet, +): boolean { + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + while (current) { + if (ancestorIndexes.has(current.index)) return true; + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return false; } /** diff --git a/src/contracts/interaction.ts b/src/contracts/interaction.ts index 8f8dfeef0..c68b7a1e8 100644 --- a/src/contracts/interaction.ts +++ b/src/contracts/interaction.ts @@ -154,13 +154,17 @@ export type SettleObservation = { * change-only diff omits refs for elements that did not change — after a * modal dismiss the diff shows only removals, and the next button to press * (already on screen, untouched) is absent from the response. `tail` lists - * the settled tree's remaining hittable, uncovered interactive elements so - * the response stays actionable without that extra round trip. Attached - * ONLY when `diff` carries zero added-line refs (the modal-dismiss/ - * toast-only signature) — a diff with fresh added refs already hands the - * next target, so the tail would be pure byte cost. Refs already present on - * `diff`'s added lines are excluded. Capped; `tailTruncated` marks when - * candidates exceeded the cap. + * the settled tree's remaining uncovered interactive elements (excluding + * structural application/window chrome and the keyboard window's chrome) + * so the response stays actionable without that extra round trip. Attached + * ONLY when `diff` carries zero added-line refs naming a NEW target (the + * modal-dismiss/toast-only/fill signature) — a diff whose added refs hand + * the next target already pays its way, so the tail would be pure byte + * cost. Keyboard-chrome refs and self-echo refs (added lines whose node + * contains the action point: the acted-on element re-describing itself, + * e.g. a filled field re-labeled with its new value) do not count as new + * targets. Refs already present on `diff`'s added lines are excluded. + * Capped; `tailTruncated` marks when candidates exceeded the cap. */ tail?: SettleTailEntry[]; tailTruncated?: true; diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index ed1649fcb..a56769a0d 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -61,7 +61,14 @@ const SETTLED_NODES = [ ]; // A dialog dismiss: Continue survives unchanged underneath, Cancel is the -// dialog's own button (dismissed with it) — a removals-only diff. +// dialog's own button (dismissed with it) — a removals-only diff. Application +// and Window carry no `hittable` field (their normal shape) and Continue +// carries no `hittable` field either, matching #1167's post-merge benchmark +// finding: right after a dismiss animation, real buttons commonly report +// `hittable: false`/undefined, not `true`. This is the flagship regression +// case — the old `hittable === true` tail bar let Application/Window through +// (their full-screen root frame usually computes hittable) while dropping +// Continue, the only actionable target. const MODAL_BEFORE_NODES = [ { index: 0, @@ -72,14 +79,19 @@ const MODAL_BEFORE_NODES = [ { index: 1, parentIndex: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 2, + parentIndex: 1, type: 'Button', label: 'Continue', - hittable: true, rect: { x: 100, y: 300, width: 200, height: 44 }, }, { - index: 2, - parentIndex: 0, + index: 3, + parentIndex: 1, type: 'Button', label: 'Cancel', hittable: true, @@ -97,9 +109,14 @@ const MODAL_DISMISSED_NODES = [ { index: 1, parentIndex: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 2, + parentIndex: 1, type: 'Button', label: 'Continue', - hittable: true, rect: { x: 100, y: 300, width: 200, height: 44 }, }, ]; @@ -113,6 +130,15 @@ function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry }; } +function typeEntry(x: number, y: number): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.type', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x, y }, + }; +} + function tapEntry(x: number, y: number): ProviderScenarioProviderEntry { return { command: 'ios.runner.tap', @@ -297,7 +323,7 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch tapEntry(200, 422), snapshotEntry(MODAL_DISMISSED_NODES), snapshotEntry(MODAL_DISMISSED_NODES), - // press @e2 (the Continue ref from the tail): tap on the stored tree. + // press @e3 (the Continue ref from the tail): tap on the stored tree. tapEntry(200, 322), ]); const appleRunnerProvider = createAppleRunnerProviderFromTranscript( @@ -347,18 +373,22 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch }; assert.ok(settle, 'press --settle must return a settle observation'); assert.equal(settle.settled, true); - assert.deepEqual(settle.diff?.summary, { additions: 0, removals: 1, unchanged: 2 }); + assert.deepEqual(settle.diff?.summary, { additions: 0, removals: 1, unchanged: 3 }); assert.equal( settle.diff?.lines.some((line) => line.kind === 'added'), false, ); - assert.deepEqual(settle.tail, [{ ref: 'e2', role: 'button', label: 'Continue' }]); + // Application/Window survive unchanged too but are structural chrome, + // not a next actionable target — the tail excludes them and surfaces + // only the real button, even though it carries no `hittable: true` flag + // (see the MODAL_BEFORE_NODES/MODAL_DISMISSED_NODES comment). + assert.deepEqual(settle.tail, [{ ref: 'e3', role: 'button', label: 'Continue' }]); assert.equal(settle.tailTruncated, undefined); assert.equal(typeof settle.refsGeneration, 'number'); // The tail's ref acts directly on the stored settled tree, same as an // added-line ref would. - const followUp = await daemon.callCommand('press', ['@e2'], {}); + const followUp = await daemon.callCommand('press', ['@e3'], {}); const followUpData = assertRpcOk(followUp); assert.equal(followUpData.warning, undefined); assert.equal(followUpData.x, 200); @@ -368,3 +398,335 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch }, ); }); + +// #1167 post-merge benchmark, Bug B: filling a field summons the iOS keyboard, +// and the keyboard chrome (real XCUIElementTypeButton nodes like shift/return/ +// Next keyboard/Dictate — not Key nodes) used to show up as fresh added-line +// refs on the settled diff. Those refs defeated the "diff has zero added refs" +// tail trigger, so exactly the post-fill case the tail exists for never fired. +// +// Both fixtures are TRIMMED REAL CAPTURES (iPhone 17 Pro simulator, iOS 26, +// July 2026, org.reactnavigation.playground rne://stack-prevent-remove Input +// screen; `snapshot -i --json` before and after `fill @e6 "hello" --settle`, +// with the 31-key block reduced to 2 representative keys and rects rounded). +// The load-bearing real-world facts they preserve: +// - The keyboard renders in its OWN window; the "Next keyboard" and "Dictate" +// candidate-bar buttons are SIBLINGS of the [Keyboard] container's wrapper, +// NOT its descendants (an earlier hand-built fixture modeled them one hop +// under the container and hid the sibling-branch bug on real hardware). +// - The app's main window does not survive the interactive-only settled +// capture; app content re-parents onto the Application node. +// - The filled field re-labels itself ("hello") and its ancestor wrappers +// inherit that label, so the diff carries self-echo added refs. +const FILL_BEFORE_NODES = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'React Navigation Example', + hittable: true, + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Window', + hittable: true, + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Other', + label: 'Input', + rect: { x: 0, y: 0, width: 402, height: 117 }, + }, + { + index: 3, + depth: 4, + parentIndex: 2, + type: 'Button', + label: 'Home, back', + rect: { x: 10, y: 64, width: 44, height: 44 }, + }, + { + index: 4, + depth: 3, + parentIndex: 0, + type: 'ScrollView', + label: 'Discard and go back', + rect: { x: 0, y: 145, width: 402, height: 667 }, + }, + { + index: 5, + depth: 4, + parentIndex: 4, + type: 'TextField', + rect: { x: 12, y: 129, width: 377, height: 41 }, + }, + { + index: 6, + depth: 4, + parentIndex: 4, + type: 'Button', + label: 'Discard and go back', + rect: { x: 12, y: 182, width: 378, height: 40 }, + }, + { + index: 7, + depth: 4, + parentIndex: 4, + type: 'Button', + label: 'Push Article', + rect: { x: 12, y: 234, width: 378, height: 40 }, + }, +]; + +const FILL_SETTLED_NODES = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'React Navigation Example', + hittable: true, + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Window', + label: 'Next keyboard', + hittable: true, + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 2, + depth: 3, + parentIndex: 1, + type: 'Other', + label: 'Next keyboard', + hittable: true, + rect: { x: 0, y: 566, width: 402, height: 308 }, + }, + { + index: 3, + depth: 4, + parentIndex: 2, + type: 'Other', + label: 'Padding-Left', + rect: { x: 0, y: 583, width: 402, height: 233 }, + }, + { + index: 4, + depth: 5, + parentIndex: 3, + type: 'Keyboard', + label: 'Padding-Left', + rect: { x: 0, y: 583, width: 402, height: 233 }, + }, + { + index: 5, + depth: 7, + parentIndex: 4, + type: 'Key', + label: 'q', + rect: { x: 5, y: 590, width: 39, height: 54 }, + }, + { + index: 6, + depth: 7, + parentIndex: 4, + type: 'Key', + label: 'space', + rect: { x: 103, y: 752, width: 197, height: 54 }, + }, + { + index: 7, + depth: 7, + parentIndex: 4, + type: 'Button', + label: 'shift', + identifier: 'shift', + rect: { x: 5, y: 698, width: 51, height: 54 }, + }, + { + index: 8, + depth: 7, + parentIndex: 4, + type: 'Button', + label: 'return', + identifier: 'Return', + rect: { x: 301, y: 752, width: 99, height: 54 }, + }, + { + index: 9, + depth: 5, + parentIndex: 2, + type: 'Button', + label: 'Next keyboard', + value: 'Polski', + hittable: true, + rect: { x: 8, y: 806, width: 68, height: 69 }, + }, + { + index: 10, + depth: 5, + parentIndex: 2, + type: 'Button', + label: 'Dictate', + identifier: 'dictation', + rect: { x: 325, y: 805, width: 68, height: 69 }, + }, + { + index: 11, + depth: 1, + parentIndex: 0, + type: 'Other', + label: 'Input', + rect: { x: 0, y: 0, width: 402, height: 117 }, + }, + { + index: 12, + depth: 4, + parentIndex: 11, + type: 'Button', + label: 'Home, back', + rect: { x: 10, y: 64, width: 44, height: 44 }, + }, + { + index: 13, + depth: 2, + parentIndex: 0, + type: 'Other', + label: 'Discard and go back', + rect: { x: 0, y: 117, width: 402, height: 757 }, + }, + { + index: 14, + depth: 3, + parentIndex: 13, + type: 'ScrollView', + label: 'hello', + rect: { x: 0, y: 145, width: 402, height: 667 }, + }, + { + index: 15, + depth: 6, + parentIndex: 14, + type: 'TextField', + label: 'hello', + value: 'hello', + rect: { x: 12, y: 129, width: 377, height: 41 }, + }, + { + index: 16, + depth: 5, + parentIndex: 14, + type: 'Button', + label: 'Discard and go back', + rect: { x: 12, y: 182, width: 378, height: 40 }, + }, + { + index: 17, + depth: 5, + parentIndex: 14, + type: 'Button', + label: 'Push Article', + rect: { x: 12, y: 234, width: 378, height: 40 }, + }, +]; + +test('Provider-backed integration fill --settle summoning the keyboard still attaches the unchanged interactive tail', async () => { + const runnerTranscript = createProviderTranscript([ + // snapshot -i: issues refs + snapshotEntry(FILL_BEFORE_NODES), + // fill @e6 'hello' --settle: the ref resolves on the stored tree (no + // fresh capture), the runner types, then the settle loop captures. The + // keyboard window appears and the field re-labels itself. + typeEntry(201, 149), + snapshotEntry(FILL_SETTLED_NODES), + snapshotEntry(FILL_SETTLED_NODES), + // press @e17 (the "Discard and go back" ref from the tail): tap on the + // stored settled tree. + tapEntry(201, 202), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + + const snapshot = await daemon.callCommand('snapshot', [], { + snapshotInteractiveOnly: true, + }); + assertRpcOk(snapshot); + + const fill = await daemon.callCommand('fill', ['@e6', 'hello'], { + settle: true, + settleQuietMs: 25, + timeoutMs: 10_000, + }); + const fillData = assertRpcOk(fill); + const settle = fillData.settle as { + settled: boolean; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + tail?: Array<{ ref: string; role: string; label?: string }>; + tailTruncated?: boolean; + }; + assert.ok(settle, 'fill --settle must return a settle observation'); + assert.equal(settle.settled, true); + // Added: the keyboard container signal line plus the filled field's + // self-echo relabels (wrapper, scroll-area, text-field now "hello"). + // The keyboard window, its wrappers, keys, shift/return, and the + // candidate-bar siblings (Next keyboard/Dictate) all collapse. + assert.deepEqual(settle.diff?.summary, { additions: 4, removals: 3, unchanged: 5 }); + const texts = settle.diff?.lines.map((line) => line.text).join('\n') ?? ''; + assert.match(texts, /\[keyboard\]/); + assert.ok(!/shift|return|Dictate|Next keyboard|\[key\]/.test(texts)); + // Chrome and self-echo added refs do not count as "the diff already + // handed back a target": the trigger still fires and the tail lists + // the screen's real controls. + assert.deepEqual(settle.tail, [ + { ref: 'e12', role: 'other', label: 'Input' }, + { ref: 'e13', role: 'button', label: 'Home, back' }, + { ref: 'e17', role: 'button', label: 'Discard and go back' }, + { ref: 'e18', role: 'button', label: 'Push Article' }, + ]); + assert.equal(settle.tailTruncated, undefined); + assert.equal(typeof settle.refsGeneration, 'number'); + + const followUp = await daemon.callCommand('press', ['@e17'], {}); + const followUpData = assertRpcOk(followUp); + assert.equal(followUpData.warning, undefined); + assert.equal(followUpData.x, 201); + assert.equal(followUpData.y, 202); + + runnerTranscript.assertComplete(); + }, + ); +});