From e14684716b6965f2f2c1fb89f5e7a2d2b2ac15f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 15:25:04 +0200 Subject: [PATCH] chore: delete dead tap-by-text runner arm and settledAfterMs; hint on tiny stable trees - Remove the runner .tap text arm: the daemon only ever sends selectorKey/selectorValue or x/y taps (src/platforms/apple/interactions.ts), and daemon+runner ship in lockstep. findElement(app:text:) stays for the findText probe. - Drop settledAfterMs from the wait stable result: it always equaled waitedMs and never shipped in a release (v0.18.3 predates #1059). - Add a loading hint when wait stable settles on a tree with fewer than 5 nodes (#1078): stability on a nearly-empty tree is a weak readiness signal. Refs #1078 --- .../RunnerTests+CommandExecution.swift | 14 +--- .../interaction/runtime/selector-read.test.ts | 84 ++++++++++++++++++- .../interaction/runtime/selector-read.ts | 13 ++- src/daemon/selector-recording.ts | 2 +- 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 49b880d7dc..de08b2d746 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -522,18 +522,6 @@ extension RunnerTests { } return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found")) } - if let text = command.text { - if let element = findElement(app: activeApp, text: text) { - let (timing, outcome) = performGesture(activeApp) { - activateElement(app: activeApp, element: element, action: "tap by text") - } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse(message: "tapped", timing: timing) - } - return Response(ok: false, error: ErrorPayload(message: "element not found")) - } if let x = command.x, let y = command.y { var fallback: GestureFallback? if command.synthesized == true { @@ -557,7 +545,7 @@ extension RunnerTests { fallback: fallback ) } - return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y")) + return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y")) case .mouseClick: guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y")) diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index d110c54d22..49e0d36585 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -367,11 +367,93 @@ test('runtime wait stable settles after two unchanged captures', async () => { if (result.kind === 'stable') { assert.equal(result.captures, 3); assert.equal(result.nodeCount, snapshot.nodes.length); - assert.equal(result.settledAfterMs, result.waitedMs); } assert.equal(captures, 3); }); +test('runtime wait stable hints when it settles on a nearly-empty tree', async () => { + const tinySnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'One', + rect: { x: 0, y: 0, width: 10, height: 10 }, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Two', + rect: { x: 0, y: 20, width: 10, height: 10 }, + }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Three', + rect: { x: 0, y: 40, width: 10, height: 10 }, + }, + ]); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: tinySnapshot }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: tinySnapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + assert.equal(result.kind, 'stable'); + if (result.kind === 'stable') { + assert.equal(result.nodeCount, 3); + assert.equal( + result.hint, + 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.', + ); + } +}); + +test('runtime wait stable omits the loading hint for a normal-sized tree', async () => { + const normalSnapshot = makeSnapshotState( + Array.from({ length: 6 }, (_, index) => ({ + index, + depth: 0, + type: 'Button', + label: `Item ${index}`, + rect: { x: 0, y: index * 20, width: 10, height: 10 }, + })), + ); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: normalSnapshot }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: normalSnapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 }, + }); + + assert.equal(result.kind, 'stable'); + if (result.kind === 'stable') { + assert.equal(result.nodeCount, 6); + assert.equal('hint' in result, false); + } +}); + test('runtime wait stable requires quiet captures after instability before settling', async () => { const snapshot = selectorSnapshot(); const changedSnapshot = makeSnapshotState([ diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 3947b6be1e..0ac338228d 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -123,9 +123,9 @@ export type WaitCommandResult = | { kind: 'stable'; waitedMs: number; - settledAfterMs: number; captures: number; nodeCount: number; + hint?: string; }; export type WaitForTextCommandOptions = CommandContext & @@ -154,6 +154,9 @@ export function ref(refInput: string, options: { fallbackLabel?: string } = {}): const DEFAULT_TIMEOUT_MS = 10_000; const POLL_INTERVAL_MS = 300; const DEFAULT_QUIET_MS = 500; +// Below this node count a settled tree is suspicious: real app surfaces have +// more than a handful of accessibility nodes, splash/loading screens do not. +const TINY_STABLE_TREE_NODE_COUNT = 5; export const findCommand: RuntimeCommand = async ( runtime, @@ -559,9 +562,15 @@ async function waitForStable( return { kind: 'stable', waitedMs: nowMs - start, - settledAfterMs: nowMs - start, captures, nodeCount: lastNodeCount, + // A settled-but-tiny tree usually means a splash/loading surface, not + // real content: stability alone is a weak readiness signal there. + ...(lastNodeCount < TINY_STABLE_TREE_NODE_COUNT + ? { + hint: 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.', + } + : {}), }; } await sleep(runtime, POLL_INTERVAL_MS); diff --git a/src/daemon/selector-recording.ts b/src/daemon/selector-recording.ts index 468b15286a..1536c92363 100644 --- a/src/daemon/selector-recording.ts +++ b/src/daemon/selector-recording.ts @@ -70,9 +70,9 @@ export function toDaemonWaitData(result: Record): Record