Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -522,18 +522,6 @@ extension RunnerTests {
}
return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
}
if let text = command.text {
if let element = findElement(app: activeApp, text: text) {
let (timing, outcome) = performGesture(activeApp) {
activateElement(app: activeApp, element: element, action: "tap by text")
}
if let response = unsupportedResponse(for: outcome) {
return response
}
return gestureResponse(message: "tapped", timing: timing)
}
return Response(ok: false, error: ErrorPayload(message: "element not found"))
}
if let x = command.x, let y = command.y {
var fallback: GestureFallback?
if command.synthesized == true {
Expand All @@ -557,7 +545,7 @@ extension RunnerTests {
fallback: fallback
)
}
return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y"))
case .mouseClick:
guard let x = command.x, let y = command.y else {
return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y"))
Expand Down
84 changes: 83 additions & 1 deletion src/commands/interaction/runtime/selector-read.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -367,11 +367,93 @@ test('runtime wait stable settles after two unchanged captures', async () => {
if (result.kind === 'stable') {
assert.equal(result.captures, 3);
assert.equal(result.nodeCount, snapshot.nodes.length);
assert.equal(result.settledAfterMs, result.waitedMs);
}
assert.equal(captures, 3);
});

test('runtime wait stable hints when it settles on a nearly-empty tree', async () => {
const tinySnapshot = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Button',
label: 'One',
rect: { x: 0, y: 0, width: 10, height: 10 },
},
{
index: 1,
depth: 0,
type: 'Button',
label: 'Two',
rect: { x: 0, y: 20, width: 10, height: 10 },
},
{
index: 2,
depth: 0,
type: 'Button',
label: 'Three',
rect: { x: 0, y: 40, width: 10, height: 10 },
},
]);
const device = createAgentDevice({
backend: {
platform: 'ios',
captureSnapshot: async () => ({ snapshot: tinySnapshot }),
} satisfies AgentDeviceBackend,
artifacts: createLocalArtifactAdapter(),
sessions: createMemorySessionStore([{ name: 'default', snapshot: tinySnapshot }]),
policy: localCommandPolicy(),
clock: createFakeClock(),
});

const result = await device.selectors.wait({
session: 'default',
target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 },
});

assert.equal(result.kind, 'stable');
if (result.kind === 'stable') {
assert.equal(result.nodeCount, 3);
assert.equal(
result.hint,
'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.',
);
}
});

test('runtime wait stable omits the loading hint for a normal-sized tree', async () => {
const normalSnapshot = makeSnapshotState(
Array.from({ length: 6 }, (_, index) => ({
index,
depth: 0,
type: 'Button',
label: `Item ${index}`,
rect: { x: 0, y: index * 20, width: 10, height: 10 },
})),
);
const device = createAgentDevice({
backend: {
platform: 'ios',
captureSnapshot: async () => ({ snapshot: normalSnapshot }),
} satisfies AgentDeviceBackend,
artifacts: createLocalArtifactAdapter(),
sessions: createMemorySessionStore([{ name: 'default', snapshot: normalSnapshot }]),
policy: localCommandPolicy(),
clock: createFakeClock(),
});

const result = await device.selectors.wait({
session: 'default',
target: { kind: 'stable', quietMs: 500, timeoutMs: 10_000 },
});

assert.equal(result.kind, 'stable');
if (result.kind === 'stable') {
assert.equal(result.nodeCount, 6);
assert.equal('hint' in result, false);
}
});

test('runtime wait stable requires quiet captures after instability before settling', async () => {
const snapshot = selectorSnapshot();
const changedSnapshot = makeSnapshotState([
Expand Down
13 changes: 11 additions & 2 deletions src/commands/interaction/runtime/selector-read.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ export type WaitCommandResult =
| {
kind: 'stable';
waitedMs: number;
settledAfterMs: number;
captures: number;
nodeCount: number;
hint?: string;
};

export type WaitForTextCommandOptions = CommandContext &
Expand Down Expand Up @@ -154,6 +154,9 @@ export function ref(refInput: string, options: { fallbackLabel?: string } = {}):
const DEFAULT_TIMEOUT_MS = 10_000;
const POLL_INTERVAL_MS = 300;
const DEFAULT_QUIET_MS = 500;
// Below this node count a settled tree is suspicious: real app surfaces have
// more than a handful of accessibility nodes, splash/loading screens do not.
const TINY_STABLE_TREE_NODE_COUNT = 5;

export const findCommand: RuntimeCommand<FindReadCommandOptions, FindReadCommandResult> = async (
runtime,
Expand Down Expand Up @@ -559,9 +562,15 @@ async function waitForStable(
return {
kind: 'stable',
waitedMs: nowMs - start,
settledAfterMs: nowMs - start,
captures,
nodeCount: lastNodeCount,
// A settled-but-tiny tree usually means a splash/loading surface, not
// real content: stability alone is a weak readiness signal there.
...(lastNodeCount < TINY_STABLE_TREE_NODE_COUNT
? {
hint: 'Settled on a nearly-empty tree — the app may still be loading. Wait for specific content (wait text ...) before interacting.',
}
: {}),
};
}
await sleep(runtime, POLL_INTERVAL_MS);
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/selector-recording.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ export function toDaemonWaitData(result: Record<string, unknown>): Record<string
waitedMs: result.waitedMs,
...(typeof result.text === 'string' ? { text: result.text } : {}),
...(typeof result.selector === 'string' ? { selector: result.selector } : {}),
...(typeof result.settledAfterMs === 'number' ? { settledAfterMs: result.settledAfterMs } : {}),
...(typeof result.captures === 'number' ? { captures: result.captures } : {}),
...(typeof result.nodeCount === 'number' ? { nodeCount: result.nodeCount } : {}),
...(typeof result.hint === 'string' ? { hint: result.hint } : {}),
};
}

Expand Down
Loading