feat(interaction): report readiness wait on a successful press, click, or longpress

packages/contracts/src/interaction.ts

@@ -181,8 +181,18 @@ export type ResolvedInteractionTarget =
       resolution?: ResolutionDisclosure;
       recordingTarget?: RecordingTargetOverride;
       preAction?: SurfaceScopedNodes;
+      readiness?: ReadinessWaitEvidence;
     };
 
+/**
+ * The wait a successful press/click/longpress spent before its target appeared. Reported only when
+ * more than one capture was needed, so its presence means the step would have failed without the wait.
+ */
+export type ReadinessWaitEvidence = {
+  polls: number;
+  waitedMs: number;
+};
+
 /**
  * A post-action capture that describes a DIFFERENT surface than the pre-action baseline (#2438): an
  * in-place iOS system surface (a web sign-in or Apple Pay sheet, hosted out of the app's process)
@@ -354,6 +364,7 @@ type TouchResponseDataBase = {
   evidence?: InteractionEvidence;
   settle?: SettleObservation;
   resolution?: ResolutionDisclosure;
+  readiness?: ReadinessWaitEvidence;
   cost?: ResponseCost;
   /** Direct iOS Maestro coordinate-fallback signals. */
   maestroNonHittableCoordinateFallbackAllowed?: boolean;

src/__tests__/client-interactions-readiness.test.ts

@@ -17,3 +17,17 @@ test('readinessTimeoutMs on press/click/longpress reaches the request flags', as
   assert.equal(setup.calls[1]?.flags?.readinessTimeoutMs, 1_500);
   assert.equal(setup.calls[2]?.flags?.readinessTimeoutMs, 900);
 });
+
+test('a press that waited returns the daemon data.readiness untouched', async () => {
+  const readiness = { polls: 2, waitedMs: 180 };
+  const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } }));
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  const pressed = await client.interactions.press({ selector: 'label=Foo' });
+  const clicked = await client.interactions.click({ selector: 'label=Foo' });
+  const longPressed = await client.interactions.longPress({ selector: 'label=Foo' });
+
+  for (const result of [pressed, clicked, longPressed]) {
+    assert.deepEqual((result as { readiness?: unknown }).readiness, readiness);
+  }
+});

src/commands/interaction/runtime/resolution.ts

@@ -5,6 +5,7 @@ import { readinessScheduleFor } from '@agent-device/selectors/selector-pipeline-
 import {
   attemptSelectorResolution,
   pollForSelectorReadiness,
+  type SelectorReadinessWait,
   selectorInteractionFailure,
 } from './selector-readiness.ts';
 import {
@@ -138,6 +139,7 @@ async function resolveSelectorInteractionTarget(
   const selectorExpression = target.selector;
   let capture: InteractionSnapshot;
   let resolved: SelectorResolution | null;
+  let readiness: SelectorReadinessWait | undefined;
   const readinessSchedule = readinessScheduleFor(params.pipeline.poll, params.readinessTimeoutMs);
   if (!readinessSchedule) {
     const attempt = await attemptSelectorResolution(runtime, options, selectorExpression, params);
@@ -162,6 +164,7 @@ async function resolveSelectorInteractionTarget(
     );
     capture = ready.capture;
     resolved = ready.resolved;
+    readiness = ready.readiness;
   }
   // #1542: see the ref-target twin in ref-target-resolution.ts.
   const selected = resolved;
@@ -188,6 +191,7 @@ async function resolveSelectorInteractionTarget(
     kind: 'selector',
     point,
     target: { kind: 'selector', selector: resolved.selector },
+    ...(readiness ? { readiness } : {}),
     ...describeResolvedInteractionNode(
       runtime,
       visibleNode,

src/commands/interaction/runtime/selector-readiness.test.ts

@@ -60,12 +60,28 @@ test('runtime press with readinessTimeoutMs polls until the target appears', asy
 
   assert.equal(result.kind, 'selector');
   assert.equal(result.node?.label, 'Continue');
+  assert.equal(result.kind === 'selector' ? result.readiness?.polls : undefined, 2);
   assert.ok(
     captures >= 4,
     `expected at least 4 captures before the target appeared, got ${captures}`,
   );
 });
 
+test('runtime press that resolves on the first capture reports no readiness despite a budget', async () => {
+  const device = createInteractionDevice(makeSnapshotState([CONTINUE_BUTTON]), {
+    clock: createFakeClock(),
+    tap: async () => ({ ok: true }),
+  });
+
+  const result = await device.interactions.press(selector('label=Continue'), {
+    session: 'default',
+    readinessTimeoutMs: 2_000,
+  });
+
+  assert.equal(result.kind, 'selector');
+  assert.equal('readiness' in result, false);
+});
+
 test('runtime press caps a readinessTimeoutMs larger than the row maxTimeoutMs at 2_000ms', async () => {
   const device = createInteractionDevice(makeSnapshotState([]), {
     clock: createFakeClock(),

src/commands/interaction/runtime/selector-readiness.ts

@@ -44,8 +44,13 @@ export type SelectorResolutionAttempt = {
 export type ResolvedSelectorAttempt = {
   capture: InteractionSnapshot;
   resolved: SelectorResolution;
+  /** The wait that preceded the hit; absent when the first capture resolved. */
+  readiness?: SelectorReadinessWait;
 };
 
+/** What a successful readiness wait reports: present only when more than one capture was needed. */
+export type SelectorReadinessWait = Pick<SelectorReadinessDetails, 'polls' | 'waitedMs'>;
+
 /** The readiness poll's evidence, attached to a target-not-found failure only (never to a refusal). */
 export type SelectorReadinessDetails = {
   polls: number;
@@ -272,7 +277,14 @@ export async function pollForSelectorReadiness(
     ...(runtime.clock ? { clock: runtime.clock } : {}),
     phase: 'interaction_target_readiness',
   });
-  if (observed.kind === 'done') return observed.result;
+  if (observed.kind === 'done') {
+    return observed.polls.length > 1
+      ? {
+          ...observed.result,
+          readiness: { polls: observed.polls.length, waitedMs: observed.waitedMs },
+        }
+      : observed.result;
+  }
   if (observed.kind === 'failed') throw withSparseReadiness(observed.error, observed);
   throw await readinessExhaustedFailure(runtime, selectorExpression, params, observed);
 }

src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts

@@ -2,7 +2,11 @@ import { test, expect, vi, beforeEach } from 'vitest';
 import { attachRefs } from '@agent-device/kernel/snapshot';
 import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts';
 import { handleInteractionCommands } from '../../index.ts';
-import { transformTouchResponseData } from '../interaction-touch-response.ts';
+import {
+  buildInteractionResponseData,
+  transformTouchResponseData,
+} from '../interaction-touch-response.ts';
+import type { PressCommandResult } from '@agent-device/contracts/interaction';
 import {
   getRuntimeBindings,
   mockFillPoint,
@@ -82,6 +86,41 @@ test('commandless longpress response omits internal interaction diagnostics', ()
   ).toEqual({ completedSteps: 1 });
 });
 
+function selectorResult(readiness?: { polls: number; waitedMs: number }): PressCommandResult {
+  return {
+    kind: 'selector',
+    point: { x: 10, y: 20 },
+    target: { kind: 'selector', selector: 'label=Continue' },
+    node: { ref: 'e1', index: 0, type: 'Button', label: 'Continue' },
+    selectorChain: ['label=Continue'],
+    warning: 'earlier warning',
+    ...(readiness ? { readiness } : {}),
+  };
+}
+
+const WAITED_SELECTOR_RESULT = selectorResult({ polls: 3, waitedMs: 400 });
+
+test('the response builder reports the resolved wait and keeps the prior warning', () => {
+  const { responseData, result } = buildInteractionResponseData({
+    source: { kind: 'runtime', result: WAITED_SELECTOR_RESULT },
+    referenceFrame: undefined,
+    staleRefsWarning: 'stale refs',
+  });
+
+  expect(responseData.readiness).toEqual({ polls: 3, waitedMs: 400 });
+  expect(responseData.warning).toBe('earlier warning stale refs');
+  expect(result.readiness).toBeUndefined();
+});
+
+test('the response builder omits readiness when nothing waited', () => {
+  const { responseData } = buildInteractionResponseData({
+    source: { kind: 'runtime', result: selectorResult() },
+    referenceFrame: undefined,
+  });
+
+  expect('readiness' in responseData).toBe(false);
+});
+
 test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => {
   const sessionStore = makeSessionStore();
   const sessionName = 'verify-press';

src/daemon/interaction/internal/interaction-touch-response.ts

@@ -234,6 +234,7 @@ export function buildInteractionResponseData(params: {
     visualization.warning = warning;
     responseData.warning = warning;
   }
+  if ('readiness' in result && result.readiness) responseData.readiness = result.readiness;
   return { result: visualization, responseData, ...recordedTargetCapture(result) };
 }
 

src/mcp/__tests__/command-tools-readiness.test.ts

@@ -0,0 +1,20 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import { createAgentDeviceClient } from '../../agent-device-client.ts';
+import { createTransport } from '../../__tests__/client-transport-fixture.ts';
+import { createCommandToolExecutor } from '../command-tools.ts';
+
+test('MCP press returns the daemon data.readiness in structuredContent', async () => {
+  const readiness = { polls: 2, waitedMs: 180 };
+  const setup = createTransport(async () => ({ ok: true, data: { ref: '@e1', readiness } }));
+  const executor = createCommandToolExecutor({
+    createClient: () => createAgentDeviceClient(setup.config, { transport: setup.transport }),
+  });
+
+  const result = await executor.execute('press', {
+    target: { kind: 'selector', selector: 'label=Continue' },
+  });
+
+  assert.equal(result.isError, false);
+  assert.deepEqual(result.structuredContent?.readiness, readiness);
+});

test/integration/provider-scenarios/press-target-readiness.test.ts

@@ -142,6 +142,41 @@ test('press waits for a selector missing on the first two captures, then taps on
   );
 });
 
+test('press whose target appears on the third poll reports readiness with polls 3 on the success response', async () => {
+  await withPressReadinessDaemon(
+    [
+      snapshotEntry(APPLICATION_ONLY_NODES),
+      snapshotEntry(APPLICATION_ONLY_NODES),
+      snapshotEntry(APPLICATION_ONLY_NODES),
+      snapshotEntry(APPLICATION_ONLY_NODES),
+      repeatSnapshotEntry(CONTINUE_BUTTON_NODES),
+      tapEntry(200, 322),
+    ],
+    async (daemon) => {
+      const press = await daemon.callCommand('press', ['label=Continue'], {
+        readinessTimeoutMs: 2_000,
+      });
+      const data = assertRpcOk(press);
+      const readiness = data.readiness as { polls: number; waitedMs: number } | undefined;
+      assert.equal(readiness?.polls, 3);
+      assert.equal(typeof readiness?.waitedMs, 'number');
+    },
+  );
+});
+
+test('press whose first capture hits carries no readiness field even with a readiness budget', async () => {
+  await withPressReadinessDaemon(
+    [snapshotEntry(CONTINUE_BUTTON_NODES), tapEntry(200, 322)],
+    async (daemon) => {
+      const press = await daemon.callCommand('press', ['label=Continue'], {
+        readinessTimeoutMs: 2_000,
+      });
+      const data = assertRpcOk(press);
+      assert.equal('readiness' in data, false);
+    },
+  );
+});
+
 test('press fails with the standard no-match error, carrying readiness evidence, when the target never appears', async () => {
   await withPressReadinessDaemon([repeatSnapshotEntry(APPLICATION_ONLY_NODES)], async (daemon) => {
     // No fake clock is wired into this daemon-composed runtime path (AgentDeviceRuntime.clock is

website/docs/docs/client-api.md

@@ -297,7 +297,7 @@ await client.interactions.press({
 });
 ```
 
-The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag.
+The wait is capped at 2 seconds and covers only a target that has not appeared. When the target is still missing after the wait, the error carries `error.details.readiness` with `waitedMs`, `polls`, and `end` (`expired` or `stalled`). A capture that shows an empty accessibility tree ends the wait at once with `capture_sparse` and `readiness.end: sparse`. When the command had to wait and then succeeded, the result carries `data.readiness` with `polls` and `waitedMs`. A command that found its target on the first look has no `readiness` field. A covered, off-screen, or ambiguous target fails at once, and a screen that stays unreadable for the whole wait fails with its own error; neither carries `readiness`. `readinessTimeoutMs` is not an MCP tool argument and has no CLI flag.
 
 Vega OS client support is currently VVD-only and covers device discovery, app open/close, `back`, `home`, and `tvRemote`. Physical Fire TV, capture, selector, install, logging, and performance methods report unsupported for Vega targets.
 

website/docs/docs/replay-e2e.md

@@ -62,6 +62,8 @@ agent-device replay ~/.agent-device/sessions/e2e-2026-02-09T12-00-00-000Z.ad --s
   they fail. A step recorded against a screen that was still loading passes on replay once the
   target shows up. The wait covers only a target that is not on screen yet: a target that is
   covered, off-screen, or matched by more than one element fails at once, as it does live.
+  A step that waited and then passed shows no trace of the wait in the replay output; run with
+  `--debug` to see it in the diagnostics.
 - When the target never appears, replay stops with `REPLAY_DIVERGENCE`, and
   `error.details.readiness` says how long the step waited and how many times it looked (`waitedMs`,
   `polls`, `end`). For a step recorded with a target annotation (the `# agent-device:target-v1`