fix(ios): prefer independent snapshots after gestures

Author: thymikee

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -1229,7 +1229,8 @@ extension RunnerTests {
       interactiveOnly: command.interactiveOnly ?? false,
       depth: command.depth,
       scope: command.scope,
-      raw: command.raw ?? false
+      raw: command.raw ?? false,
+      preferIndependentBackend: command.snapshotCapturePreference == "independent"
     )
     do {
       let payload: DataPayload

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -141,6 +141,7 @@ struct Command: Codable {
   let depth: Int?
   let scope: String?
   let raw: Bool?
+  let snapshotCapturePreference: String?
   let fullscreen: Bool?
   let inlineScreenshot: Bool?
   let synthesized: Bool?
@@ -390,4 +391,19 @@ struct SnapshotOptions {
   let depth: Int?
   let scope: String?
   let raw: Bool
+  let preferIndependentBackend: Bool
+
+  init(
+    interactiveOnly: Bool,
+    depth: Int?,
+    scope: String?,
+    raw: Bool,
+    preferIndependentBackend: Bool = false
+  ) {
+    self.interactiveOnly = interactiveOnly
+    self.depth = depth
+    self.scope = scope
+    self.raw = raw
+    self.preferIndependentBackend = preferIndependentBackend
+  }
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -54,6 +54,7 @@ enum SnapshotBackendKind: String, CaseIterable {
 
 enum SnapshotXCTestChannelPlanState: Equatable {
   case normal
+  case preferredIndependentBackend
   case deferredToIndependentBackend
   case boundedXCTestProbe
 }
@@ -150,16 +151,34 @@ extension RunnerTests {
   static func effectiveSnapshotCapturePlan(
     _ plan: [SnapshotBackendKind],
     xCTestChannelPenalized: Bool,
-    availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases)
+    availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases),
+    preferIndependentBackend: Bool = false
   ) -> EffectiveSnapshotCapturePlan {
-    guard xCTestChannelPenalized, plan == Self.regularVisiblePlan else {
+    guard plan == Self.regularVisiblePlan else {
       return EffectiveSnapshotCapturePlan(
         plan: plan,
         xCTestChannelState: .normal,
         treeCaptureSliceBudgetOverride: nil
       )
     }
     let availablePlan = plan.filter { availableBackends.contains($0) }
+    if preferIndependentBackend && !xCTestChannelPenalized {
+      let independentPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel }
+      if !independentPlan.isEmpty {
+        return EffectiveSnapshotCapturePlan(
+          plan: independentPlan + availablePlan.filter(\.usesXCTestAccessibilityChannel),
+          xCTestChannelState: .preferredIndependentBackend,
+          treeCaptureSliceBudgetOverride: nil
+        )
+      }
+    }
+    guard xCTestChannelPenalized else {
+      return EffectiveSnapshotCapturePlan(
+        plan: availablePlan,
+        xCTestChannelState: .normal,
+        treeCaptureSliceBudgetOverride: nil
+      )
+    }
     let recoveryPlan = availablePlan.filter { !$0.usesXCTestAccessibilityChannel }
     if !recoveryPlan.isEmpty {
       return EffectiveSnapshotCapturePlan(
@@ -205,12 +224,19 @@ extension RunnerTests {
     let effective = Self.effectiveSnapshotCapturePlan(
       plan,
       xCTestChannelPenalized: xCTestChannelPenalized,
-      availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform))
+      availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)),
+      preferIndependentBackend: options.preferIndependentBackend
     )
     let effectivePlan = effective.plan
     switch effective.xCTestChannelState {
     case .normal:
       break
+    case .preferredIndependentBackend:
+      firstFailure = (
+        "an independent accessibility backend was preferred to verify post-gesture freshness",
+        "freshness"
+      )
+      NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INDEPENDENT_BACKEND_PREFERRED bundle=%@", currentBundleId ?? "")
     case .deferredToIndependentBackend:
       firstFailure = (
         "XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen",
@@ -301,7 +327,10 @@ extension RunnerTests {
         continue
       }
 
-      let recovered = kind != effectivePlan.first || effective.xCTestChannelState != .normal
+      let recovered =
+        kind != effectivePlan.first
+        || effective.xCTestChannelState == .deferredToIndependentBackend
+        || effective.xCTestChannelState == .boundedXCTestProbe
       if recovered {
         NSLog(
           "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@",
@@ -687,6 +716,28 @@ extension RunnerTests {
     )
   }
 
+  func testEffectiveSnapshotCapturePlanPrefersIndependentBackendForFreshnessProbe() {
+    let simulatorPlan = Self.effectiveSnapshotCapturePlan(
+      Self.regularVisiblePlan,
+      xCTestChannelPenalized: false,
+      preferIndependentBackend: true
+    )
+
+    XCTAssertEqual(simulatorPlan.plan, [.privateAX, .recursiveTree, .querySweep])
+    XCTAssertEqual(simulatorPlan.xCTestChannelState, .preferredIndependentBackend)
+    XCTAssertNil(simulatorPlan.treeCaptureSliceBudgetOverride)
+
+    let physicalDevicePlan = Self.effectiveSnapshotCapturePlan(
+      Self.regularVisiblePlan,
+      xCTestChannelPenalized: false,
+      availableBackends: [.recursiveTree, .querySweep],
+      preferIndependentBackend: true
+    )
+
+    XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep])
+    XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .normal)
+  }
+
   func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() {
     defer {
       snapshotXCTestChannelPenaltyBundleId = nil

docs/adr/0004-ios-snapshot-backend-strategy.md

@@ -51,6 +51,12 @@ strategies:
   carry the response, fail explicitly instead of silently truncating the tree at a hard node count.
   If XCTest reports a real AX serialization failure, preserve that error instead of pretending the
   UI is empty.
+- **Post-gesture freshness probe**: after the daemon observes two quiet captures that still match
+  the pre-gesture surface, it may ask the regular plan to prefer an independent backend for the
+  next observations. On iOS simulators this puts private AX first while retaining the normal plan
+  as recovery; ordinary snapshots remain tree-first. The daemon also uses a fixed distrust
+  observation count, with a separate hard safety ceiling, so slow XCTest reads cannot turn a
+  contention-shaped delay into premature stale geometry acceptance.
 - **Future AX-service strategy**: treat Bluesky-class failures as evidence that XCTest is
   not a complete semantic snapshot backend. A robust semantic fix should add a host-side simulator
   accessibility backend, similar in role to existing simulator accessibility inspection tools,

packages/contracts/src/interactor-types.ts

@@ -64,6 +64,8 @@ export type ElementSelectorTapOptions = {
   expectedPoint?: Point;
 };
 
+export type SnapshotCapturePreference = 'independent';
+
 /**
  * Legacy success text retained for compatibility when the XCTest runner used
  * the Maestro non-hittable coordinate fallback. Usage itself is carried by
@@ -76,6 +78,7 @@ export type SnapshotOptions = BaseSnapshotOptions & {
   signal?: AbortSignal;
   includeRects?: boolean;
   includeHiddenContentHints?: boolean;
+  snapshotCapturePreference?: SnapshotCapturePreference;
   surface?: SessionSurface;
 };
 

src/core/dispatch-context.ts

@@ -7,6 +7,7 @@ import type {
   BackMode,
   ClickButton,
   ElementSelectorKey,
+  SnapshotCapturePreference,
   SwipePattern,
 } from '@agent-device/contracts/interaction';
 import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/platform';
@@ -36,6 +37,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
   snapshotDepth?: number;
   snapshotScope?: string;
   snapshotRaw?: boolean;
+  snapshotCapturePreference?: SnapshotCapturePreference;
   snapshotIncludeRects?: boolean;
   snapshotIncludeHiddenContentHints?: boolean;
   skipIosSimulatorBootCheck?: boolean;

src/core/dispatch.ts

@@ -639,6 +639,7 @@ async function handleSnapshotCommand(
     depth: snapshotContext.snapshotDepth,
     scope: snapshotContext.snapshotScope,
     raw: snapshotContext.snapshotRaw,
+    snapshotCapturePreference: snapshotContext.snapshotCapturePreference,
     includeRects: snapshotContext.snapshotIncludeRects,
     includeHiddenContentHints: snapshotContext.snapshotIncludeHiddenContentHints,
     surface: snapshotContext.surface,

src/daemon/__tests__/post-gesture-stabilization-verdict.test.ts

@@ -120,6 +120,35 @@ test('decidePostGestureStabilityVerdict accepts a baseline-matching signature on
   );
 });
 
+test('decidePostGestureStabilityVerdict uses the observation budget when capture latency exceeds the time cap', () => {
+  const signature = buildInteractionSurfaceSignature(pickupSnapshot().nodes);
+
+  assert.equal(
+    decidePostGestureStabilityVerdict({
+      needsBaselineDistrust: true,
+      baselineSignature: signature,
+      quietSignature: signature,
+      elapsedMs: 9_000,
+      distrustCapMs: 3_500,
+      distrustAttemptCount: 11,
+      distrustAttemptLimit: 12,
+    }),
+    'distrust',
+  );
+  assert.equal(
+    decidePostGestureStabilityVerdict({
+      needsBaselineDistrust: true,
+      baselineSignature: signature,
+      quietSignature: signature,
+      elapsedMs: 2_000,
+      distrustCapMs: 3_500,
+      distrustAttemptCount: 12,
+      distrustAttemptLimit: 12,
+    }),
+    'accept-stale',
+  );
+});
+
 // --- #1563 review, finding 1: a root-only shared overlap must trust immediately, not tax the cap ---
 
 test('decidePostGestureStabilityVerdict trusts immediately when a real scroll leaves only the application root shared (no cap tax)', () => {

src/daemon/__tests__/post-gesture-stabilization.test.ts

@@ -153,6 +153,80 @@ test('capturePostGestureStabilizedResult keeps polling past the normal deadline
   assert.ok(captureCount > 8, `expected sustained polling, saw ${captureCount} captures`);
 });
 
+test('capturePostGestureStabilizedResult preserves its distrust observation count when captures are slow (iOS)', async () => {
+  vi.useFakeTimers();
+  vi.setSystemTime(0);
+  const session = makeSession('ios');
+  session.snapshot = pickupSnapshot(500);
+  markPostGestureStabilization(session, 'scroll');
+
+  const capture = vi.fn(async () => {
+    // Simulate the XCTest channel taking 500ms per read without making the test
+    // itself wait. The current wall-clock cap accepts after roughly six reads,
+    // before it has collected the normal distrust sample count.
+    vi.setSystemTime(Date.now() + 500);
+    return pickupSnapshot(500);
+  });
+
+  const resultPromise = withDiagnosticsScope({}, async () => {
+    const result = await capturePostGestureStabilizedResult({
+      session,
+      capture,
+      readSnapshot: (snapshot) => snapshot,
+    });
+    return {
+      result,
+      staleAccepts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stale_accept']),
+    };
+  });
+
+  await vi.advanceTimersByTimeAsync(20_000);
+  const { staleAccepts } = await resultPromise;
+
+  assert.equal(staleAccepts, 1);
+  assert.ok(
+    capture.mock.calls.length >= 12,
+    `expected the distrust sample count to survive slow captures, saw ${capture.mock.calls.length}`,
+  );
+});
+
+test('capturePostGestureStabilizedResult switches to independent evidence after a baseline match (iOS)', async () => {
+  vi.useFakeTimers();
+  const session = makeSession('ios');
+  session.snapshot = pickupSnapshot(500);
+  markPostGestureStabilization(session, 'scroll');
+
+  const capture = vi.fn(async (options?: { snapshotCapturePreference?: 'independent' }) =>
+    options?.snapshotCapturePreference === 'independent'
+      ? pickupSnapshot(120)
+      : pickupSnapshot(500),
+  );
+
+  const resultPromise = withDiagnosticsScope({}, async () => {
+    const result = await capturePostGestureStabilizedResult({
+      session,
+      capture,
+      readSnapshot: (snapshot) => snapshot,
+    });
+    return {
+      result,
+      staleAccepts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stale_accept']),
+      settled: countDiagnosticEventsByPhase(['post_gesture_snapshot_stabilized']),
+    };
+  });
+
+  await vi.advanceTimersByTimeAsync(2_000);
+  const { staleAccepts, settled } = await resultPromise;
+
+  assert.equal(staleAccepts, 0);
+  assert.equal(settled, 1);
+  assert.equal(capture.mock.calls.length, 4);
+  assert.equal(capture.mock.calls[0]?.[0], undefined);
+  assert.equal(capture.mock.calls[1]?.[0], undefined);
+  assert.deepEqual(capture.mock.calls[2], [{ snapshotCapturePreference: 'independent' }]);
+  assert.deepEqual(capture.mock.calls[3], [{ snapshotCapturePreference: 'independent' }]);
+});
+
 test('capturePostGestureStabilizedResult trusts a quiet signature once content genuinely differs from the baseline (iOS)', async () => {
   vi.useFakeTimers();
   const session = makeSession('ios');

src/daemon/handlers/snapshot-capture.ts

@@ -14,6 +14,7 @@ import {
   type SnapshotBackend,
   type SnapshotState,
 } from '@agent-device/kernel/snapshot';
+import type { SnapshotCapturePreference } from '@agent-device/contracts/interaction';
 import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
 import { runMacOsSnapshotAction } from '../../platforms/apple/os/macos/helper.ts';
 import { snapshotLinux } from '../../platforms/linux/snapshot.ts';
@@ -58,6 +59,7 @@ type CaptureSnapshotParams = {
   outPath?: string;
   logPath: string;
   snapshotScope?: string;
+  snapshotCapturePreference?: SnapshotCapturePreference;
   androidFreshnessMode?: AndroidFreshnessMode;
   signal?: AbortSignal;
 };
@@ -156,7 +158,7 @@ async function captureInteractionOutcomeAwareSnapshot(
   latest = await capturePostGestureStabilizedResult({
     session,
     initial: latest,
-    capture: async () => await capturePostActionSnapshotAttempt(params),
+    capture: async (options) => await capturePostActionSnapshotAttempt(params, options),
     readSnapshot: (attempt) => attempt.snapshot,
   });
   if (outcome.change !== 'ambiguous' && latest.annotations.freshness?.staleAfterRetries !== true) {
@@ -229,6 +231,7 @@ export async function captureSnapshotData(params: CaptureSnapshotParams): Promis
       session?.trace?.outPath,
     ),
     snapshotIncludeRects: params.includeRects,
+    snapshotCapturePreference: params.snapshotCapturePreference,
     signal: params.signal,
   })) as SnapshotData;
 }
@@ -290,7 +293,7 @@ async function capturePostGestureAwareSnapshot(
 ): Promise<CaptureSnapshotResult> {
   const latest = await capturePostGestureStabilizedResult({
     session: params.session,
-    capture: async () => await capturePostActionSnapshotAttempt(params),
+    capture: async (options) => await capturePostActionSnapshotAttempt(params, options),
     readSnapshot: (attempt) => attempt.snapshot,
   });
   return {
@@ -301,12 +304,17 @@ async function capturePostGestureAwareSnapshot(
 
 async function capturePostActionSnapshotAttempt(
   params: CaptureSnapshotParams & { session: SessionState },
+  options?: { snapshotCapturePreference?: SnapshotCapturePreference },
 ): Promise<SnapshotAttempt> {
   const freshness = getActiveAndroidSnapshotFreshness(params.session);
   if (freshness && params.device.platform === 'android') {
     return await captureAndroidFreshnessAwareAttempt(params, freshness);
   }
-  return await captureSnapshotAttempt(params);
+  return await captureSnapshotAttempt(
+    options?.snapshotCapturePreference
+      ? { ...params, snapshotCapturePreference: options.snapshotCapturePreference }
+      : params,
+  );
 }
 
 async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise<SnapshotAttempt> {