From fa516838336ea47049d3a139e8dc66a9b57fd243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 8 Oct 2026 22:24:42 +0200 Subject: [PATCH 1/3] test(ios): accept the disclosed runner fallback in the depth-frontier snapshot assertion (#3328) The lane asserted a regular snapshot must not carry snapshotQuality at all, but the product's contract is a disclosure, not an absence: the AX bridge publishes no verdict, and the XCTest runner the route falls back to always stamps its serving strategy. When the bridge probe circuit is open for the app generation (a legitimate cold/loaded-host state), the runner-served healthy 'tree' verdict made the lane red twice on main (runs 37145696803, 37459193672). The assertion now keys on the typed verdict: an absent (bridge) or healthy verdict is accepted, a degraded 'recovered' or 'sparse' verdict still fails, never on the warning string. --- ...simulator-e2e-snapshot-acquisition.test.ts | 96 +++++++++++++++++++ .../live-snapshot-depth-frontier.ts | 60 ++++++++++-- 2 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts diff --git a/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts b/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts new file mode 100644 index 0000000000..d1d632a2d8 --- /dev/null +++ b/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { assertSimulatorSnapshotAcquisition } from './ios-simulator-e2e/live-snapshot-depth-frontier.ts'; + +// #3328, pinned without a simulator: the depth-frontier scenario accepts either acquisition it +// can present depth facts from — the AX bridge (which publishes no verdict) or the XCTest runner +// the route sent the capture to (which always stamps its strategy) — and rejects only a capture +// whose own typed verdict says it degraded or served nothing. The positives are the exact shapes +// of the two failing main runs (37145696803, 37459193672), whose healthy `tree` verdict the old +// assertion failed on the word `tree`. + +const HITTABLE = 'viewport-derived hittability evidence'; + +function capture(quality: unknown): { json?: any } { + return { + json: { + success: true, + data: { + nodes: [{ index: 0, depth: 0, hittable: true }], + ...(quality === undefined ? {} : { snapshotQuality: quality }), + warnings: [HITTABLE], + }, + }, + }; +} + +test('an AX-bridge capture carries no verdict and is accepted', () => { + assertSimulatorSnapshotAcquisition(capture(undefined), 'regular depth-1 snapshot'); +}); + +for (const backend of ['tree', 'queries', 'private-ax'] as const) { + test(`a runner-served capture disclosing the ${backend} strategy is accepted`, () => { + assertSimulatorSnapshotAcquisition( + capture({ state: 'healthy', backend, timing: { acquisitionMs: 64.7, presentationMs: 0.09 } }), + 'regular depth-1 snapshot', + ); + }); +} + +for (const reasonCode of ['deferred', 'requested-backend'] as const) { + test(`a pre-selected-backend recovered verdict (${reasonCode}) is accepted`, () => { + assertSimulatorSnapshotAcquisition( + capture({ state: 'recovered', backend: 'private-ax', reasonCode }), + 'full raw visible-depth snapshot', + ); + }); +} + +test('a capture that recovered to another strategy mid-capture is rejected', () => { + // The closest negative to the pre-selected pair: the same `recovered` state with a degradation + // code means the strategy the presented depth belongs to failed mid-capture, so the tree the + // frontier assertions would read is not a comparable view of the screen (#1569). + for (const reasonCode of ['capture-failed', 'presentation-failed', undefined]) { + assert.throws( + () => + assertSimulatorSnapshotAcquisition( + capture({ state: 'recovered', backend: 'tree', ...(reasonCode ? { reasonCode } : {}) }), + 'regular depth-1 snapshot', + ), + /fell back to another capture strategy mid-capture/, + ); + } +}); + +test('a sparse capture is rejected whatever strategy it names', () => { + assert.throws( + () => + assertSimulatorSnapshotAcquisition( + capture({ state: 'sparse', backend: 'tree', reasonCode: 'sparse-tree' }), + 'regular depth-1 snapshot', + ), + /no backend served this screen/, + ); +}); + +test('a capture that declares its hittability evidence missing is rejected', () => { + assert.throws( + () => + assertSimulatorSnapshotAcquisition( + { + json: { + success: true, + data: { + nodes: [{ index: 0, depth: 0 }], + warnings: [ + 'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.', + ], + }, + }, + }, + 'regular depth-1 snapshot', + ), + /must derive hittability/, + ); +}); diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index 0b6de9b4da..bd59d6dcdc 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -31,7 +31,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P '1', '--debug', ]); - assertSimulatorBridgeSnapshot(regular, 'regular depth-1 snapshot'); + assertSimulatorSnapshotAcquisition(regular, 'regular depth-1 snapshot'); const regularNodes = snapshotNodes(regular); const regularRoot = requireRoot(regularNodes, 'regular depth-1 snapshot'); const projectedChild = requireIdentifier(regularNodes, CHILD_ID, 'regular depth-1 snapshot'); @@ -59,7 +59,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P 'snapshot', '--raw', ]); - assertSimulatorBridgeSnapshot(rawFull, 'full raw visible-depth snapshot'); + assertSimulatorSnapshotAcquisition(rawFull, 'full raw visible-depth snapshot'); const rawFullNodes = snapshotNodes(rawFull); const rawChild = requireIdentifier(rawFullNodes, CHILD_ID, 'full raw visible-depth snapshot'); assert.ok( @@ -73,7 +73,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P '--depth', '1', ]); - assertSimulatorBridgeSnapshot(rawDepthOne, 'raw depth-1 visible-depth snapshot'); + assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot'); const rawDepthOneNodes = snapshotNodes(rawDepthOne); assert.equal( rawDepthOneNodes.some((node) => node.identifier === CHILD_ID), @@ -122,12 +122,54 @@ function numericDepth(node: SnapshotNode): number { return node.depth as number; } -function assertSimulatorBridgeSnapshot(result: { json?: any }, description: string): void { - assert.equal( - result.json?.data?.snapshotQuality?.backend, - undefined, - `${description} must not carry XCTest tree quality metadata: ${JSON.stringify(result)}`, - ); +/** + * Reason codes that name a PRE-SELECTED backend rather than a capture that degraded + * (`deferred`: the runner's penalty circuit; `requested-backend`: the caller asked for that + * strategy). They are the two codes the product's own quality-warning renderer exempts from any + * degradation sentence for the same reason: nothing on THIS capture went wrong. + */ +const PRE_SELECTED_REASON_CODES: ReadonlySet = new Set(['deferred', 'requested-backend']); + +/** + * The acquisition disclosure this scenario accepts before reading depth facts off the capture, + * keyed on the typed `snapshotQuality` verdict and never on the fallback warning's wording. + * + * A regular or raw capture of the fixture is served one of two ways, and each discloses itself + * here (#3328): + * + * - the host AX bridge served it. The bridge publishes no quality verdict at all + * (`presentIosSnapshotAcquisition` reports the tree it read without one), so an absent verdict IS + * the bridge's disclosure. + * - the route sent the capture to the XCTest runner instead — the bridge probe circuit is open for + * this app generation, the bridge is still being prepared, a system surface is presented — and + * the runner always stamps which of its own strategies served the payload. That disclosure is + * legitimate on a regular snapshot: the bridge is a fast path, its circuit is per app generation, + * and a runner-served capture keeps the hittability evidence this scenario asserts below. + * + * What the scenario may NOT be read from is a capture whose own verdict says it degraded or + * served nothing: `sparse` means no backend served the screen, and `recovered` with a code other + * than the two pre-selected ones means the strategy the presented depth semantics belong to failed + * mid-capture and another answered (#1569 — two strategies are not comparable views of one screen). + * Either one fails here with the response that proved it. + */ +export function assertSimulatorSnapshotAcquisition( + result: { json?: any }, + description: string, +): void { + const quality = result.json?.data?.snapshotQuality; + if (quality !== undefined) { + assert.notEqual( + quality.state, + 'sparse', + `${description} reports no backend served this screen: ${JSON.stringify(result)}`, + ); + if (quality.state === 'recovered') { + assert.ok( + PRE_SELECTED_REASON_CODES.has(quality.reasonCode), + `${description} fell back to another capture strategy mid-capture, so its presented depth is not comparable: ${JSON.stringify(result)}`, + ); + } + } assert.equal( result.json?.data?.warnings?.includes(MISSING_HITTABILITY_WARNING) ?? false, false, From ffd575ae7487cfb56186a05c1703c12c6cb06e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 8 Oct 2026 22:24:47 +0200 Subject: [PATCH 2/3] test(ios): re-issue a live-lane step whose typed verdict says observation was prevented (#2491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The replay steps in the iOS smoke job run with --retries 2 while the fixture-backed E2E step runs node --test with no retry layer, so one cold runner restart inside one 10s wait fails the whole job. Seven of the last ten main failures are this one class: a wait whose own error names wait_capture_stalled, wait_runner_restart_exhausted, or wait_readiness_exhausted and carries retriable: true — the product itself says 'retry' on those verdicts. The harness gains an opt-in per-step classifier; the iOS lane feeds it a predicate keyed on that typed conjunction only (wire retriable AND the wait taxonomy reason), never on error text. A re-attributed miss re-issues the step once; a readable miss (target absent, deadline exhausted after a readable capture, a wrong asserted value, a runner crash) stays a hard failure. Steps that already own a miss policy (allowFailure/expectFailure) are exempt so existing retry loops keep their semantics. --- ...os-simulator-e2e-step-retry-policy.test.ts | 74 +++++++++++ .../ios-simulator-e2e/live-harness.ts | 22 +++- .../live-step-retry-policy.ts | 38 ++++++ .../live-device-e2e-runtime.test.ts | 124 ++++++++++++++++++ test/integration/live-device-e2e/runtime.ts | 68 +++++++++- 5 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 test/integration/ios-simulator-e2e-step-retry-policy.test.ts create mode 100644 test/integration/ios-simulator-e2e/live-step-retry-policy.ts diff --git a/test/integration/ios-simulator-e2e-step-retry-policy.test.ts b/test/integration/ios-simulator-e2e-step-retry-policy.test.ts new file mode 100644 index 0000000000..71a892c44b --- /dev/null +++ b/test/integration/ios-simulator-e2e-step-retry-policy.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { WAIT_REASONS } from '@agent-device/contracts/wait'; +import type { CliJsonResult } from './cli-json.ts'; +import { isObservationPreventedStepMiss } from './ios-simulator-e2e/live-step-retry-policy.ts'; + +// The #2491 retry policy for the iOS live lane, pinned without a simulator: which typed wire +// verdicts the lane may re-issue one step for, and — the closest negatives — which look close but +// must fail at once. The pairing of `retriable: true` with an observation-prevented +// `details.reason` is the whole predicate; neither signal alone activates a re-issue. + +function stepResult(json: unknown, status = 1): CliJsonResult { + return { json, status, stdout: '', stderr: '' }; +} + +function waitFailure(reason: string, retriable: boolean | undefined): CliJsonResult { + return stepResult({ + error: { + code: 'COMMAND_FAILED', + details: { reason }, + ...(retriable === undefined ? {} : { retriable }), + }, + }); +} + +for (const reason of [ + WAIT_REASONS.captureStalled, + WAIT_REASONS.runnerRestartExhausted, + WAIT_REASONS.readinessExhausted, +]) { + test(`${reason} with the product's retriable verdict re-issues the step`, () => { + assert.equal(isObservationPreventedStepMiss(waitFailure(reason, true)), true); + }); +} + +test('an observation-prevented reason without the product retriable verdict does not re-issue', () => { + // The negative that keeps the predicate keyed on the conjunction: a response carrying the same + // reason but marked non-retriable (or carrying no verdict) must not buy a re-issue. + assert.equal( + isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.captureStalled, false)), + false, + ); + assert.equal( + isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.captureStalled, undefined)), + false, + ); +}); + +test('a retriable target-absent or deadline-exceeded wait does not re-issue', () => { + // The screen was readable: the target was not there, or a readable capture consumed the + // remaining budget. Re-issuing would hide a real absence, so these stay hard failures. + assert.equal(isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.targetAbsent, true)), false); + assert.equal( + isObservationPreventedStepMiss(waitFailure(WAIT_REASONS.deadlineExceeded, true)), + false, + ); +}); + +test('a non-wait retriable failure does not re-issue', () => { + // A retriable verdict on a response with no wait taxonomy reason (e.g. a leased-busy device on + // a click) is not evidence that observation of THIS step was prevented. + assert.equal( + isObservationPreventedStepMiss( + stepResult({ error: { code: 'DEVICE_IN_USE', retriable: true } }), + ), + false, + ); +}); + +test('a successful step result is never a re-attributable miss', () => { + assert.equal(isObservationPreventedStepMiss(stepResult(undefined, 0)), false); + assert.equal(isObservationPreventedStepMiss(stepResult(undefined)), false); +}); diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts index 37fc75b246..a1f4fcb16a 100644 --- a/test/integration/ios-simulator-e2e/live-harness.ts +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -13,6 +13,7 @@ import { } from '../live-device-e2e/runtime.ts'; import type { IosSimulatorBehaviorId } from './behavior-coverage.ts'; import { liveCommandsForScenario } from './coverage.ts'; +import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts'; export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts'; @@ -67,7 +68,26 @@ const harness = createLiveDeviceHarness({ writeCoverageReport, }); -export const { runScenario, runStep, sessionExists, verifyBehavior, verifyCommand } = harness; +/** + * Every iOS live step carries the observation-prevented re-issue policy (#2491): one runner + * restart or bridge rebuild re-issues its own wait instead of failing the job, while a readable + * miss (target absent, budget exhausted after a readable capture, wrong asserted value) fails at + * once exactly as before. Steps that already own a miss policy (`allowFailure` / `expectFailure`, + * like the destination-wait and cleanup retry loops) are exempt inside the harness itself. + */ +export function runStep( + context: LiveContext, + step: string, + args: string[], + options: Parameters[3] = {}, +): Promise { + return harness.runStep(context, step, args, { + reattemptInfrastructureMiss: isObservationPreventedStepMiss, + ...options, + }); +} + +export const { runScenario, sessionExists, verifyBehavior, verifyCommand } = harness; export function verifyNestedReplayCommand( context: LiveContext, diff --git a/test/integration/ios-simulator-e2e/live-step-retry-policy.ts b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts new file mode 100644 index 0000000000..9ad1f885f7 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts @@ -0,0 +1,38 @@ +import { WAIT_REASONS } from '@agent-device/contracts/wait'; +import type { CliJsonResult } from '../cli-json.ts'; + +/** + * The wait verdicts that say the observation itself was prevented — the runner restarted, + * readiness work (runner start, bridge target discovery) ate the budget, or no capture was ever + * readable. Each is a lane condition on this host, not an answer about the screen: the product + * itself publishes all three with `retriable: true` and a "then retry" hint on the wire + * (`wait-polling.ts`), and they are the observed reds of the smoke lane (run 37356199982's + * `wait_runner_restart_exhausted` with `readableCaptures: 0`). + */ +const OBSERVATION_PREVENTED_WAIT_REASONS: ReadonlySet = new Set([ + WAIT_REASONS.captureStalled, + WAIT_REASONS.runnerRestartExhausted, + WAIT_REASONS.readinessExhausted, +]); + +/** + * Whether one failed E2E step's result says the DEVICE prevented the observation, so re-issuing + * the same step is a measurement of the same question rather than a second bite at a real defect. + * + * Keyed on the conjunction of two typed signals — the wire `retriable` verdict the daemon hoists + * from its throw sites AND `error.details.reason` from the wait taxonomy — never on error text. + * Either signal alone is too wide: `retriable` also rides failures the lane must not re-issue + * blind (e.g. an app the lane itself is expected to have launched via another route), and a reason + * set copied onto a response the product marks non-retriable would retry a verdict the product + * says cannot succeed unchanged. The pairing is pinned in + * `test/integration/ios-simulator-e2e-step-retry-policy.test.ts`. + * + * `wait_target_absent` (the screen was readable and simply did not hold the target), + * `wait_deadline_exceeded` (a readable capture consumed the budget), a wrong assertion value, a + * runner crash, and any non-wait failure are all non-retriable by this predicate. + */ +export function isObservationPreventedStepMiss(result: CliJsonResult): boolean { + const error = result.json?.error; + if (!error || result.status === 0) return false; + return error.retriable === true && OBSERVATION_PREVENTED_WAIT_REASONS.has(error.details?.reason); +} diff --git a/test/integration/live-device-e2e-runtime.test.ts b/test/integration/live-device-e2e-runtime.test.ts index bf7b964a65..a9474d3f39 100644 --- a/test/integration/live-device-e2e-runtime.test.ts +++ b/test/integration/live-device-e2e-runtime.test.ts @@ -161,6 +161,130 @@ test('a successful scenario has no diagnostic capture', async () => { assert.equal(fs.existsSync(path.join(context.artifactDir, 'failed-step.txt')), false); }); +// #2491: the lane re-issues one step whose own typed miss says observation was prevented, so a +// single runner restart costs a repeat of the wait instead of the job. The classifier is injected +// so these cases pin the harness mechanism, and `live-step-retry-policy.ts`'s tests pin the +// verdicts the iOS lane feeds it. + +function reissueFixture(options: { + /** Statuses served per issue of the step; the last one repeats. */ + statuses: number[]; +}) { + const context = createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('step-reissue-'), + session: 'owned-fixture', + }); + const calls: string[][] = []; + let issues = 0; + const harness = createLiveDeviceHarness({ + behaviorsForScenario: () => [], + commandsForScenario: () => [], + commonFlags: (_current, args) => [...args, '--json'], + runCli: async (args): Promise => { + calls.push(args); + if (args[0] === 'screenshot' || args[0] === 'snapshot') { + if (args[0] === 'screenshot') fs.writeFileSync(args[1]!, 'fixture-png'); + return { status: 0, stdout: '', stderr: '', json: { success: true, data: { nodes: [] } } }; + } + issues += 1; + const status = options.statuses[Math.min(issues, options.statuses.length) - 1]!; + return status === 0 + ? { status: 0, stdout: '', stderr: '', json: { success: true } } + : { + status: 1, + stdout: '', + stderr: '', + json: { error: { code: 'COMMAND_FAILED', details: { reason: 'wait_stub' } } }, + }; + }, + writeCoverageReport: () => {}, + }); + return { context, harness, calls }; +} + +test('an observation-prevented miss re-issues the step once and the step passes', async () => { + let classified = 0; + const { context, harness, calls } = reissueFixture({ statuses: [1, 0] }); + const result = await harness.runStep(context, 'wait for text', ['wait', 'text', 'Home'], { + reattemptInfrastructureMiss: () => { + classified += 1; + return true; + }, + }); + assert.equal(result.status, 0); + assert.equal(classified, 1); + assert.equal(calls.filter((args) => args[0] === 'wait').length, 2); + assert.deepEqual( + context.stepHistory + .filter((record) => record.commandName === 'wait') + .map((record) => [record.step, record.status, record.accepted]), + [ + ['wait for text', 1, false], + ['wait for text (re-issue after observation-prevented miss)', 0, true], + ], + ); +}); + +test('a re-issued miss writes no failure evidence; the final issue owns it', async () => { + const { context, harness, calls } = reissueFixture({ statuses: [1, 1] }); + await assert.rejects( + harness.runStep(context, 'wait for text', ['wait', 'text', 'Home'], { + reattemptInfrastructureMiss: () => true, + }), + /re-issue after observation-prevented miss/, + ); + assert.equal(calls.filter((args) => args[0] === 'wait').length, 2); + // Evidence capture ran once, for the issue the step is judged on, and its report names that + // issue rather than the one the policy already accepted to repeat. + assert.equal(calls.filter((args) => args[0] === 'screenshot').length, 1); + const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'); + assert.match(report, /re-issue after observation-prevented miss/); +}); + +test('a miss the classifier refuses fails at once with the recorded evidence', async () => { + const { context, harness, calls } = reissueFixture({ statuses: [1] }); + await assert.rejects( + harness.runStep(context, 'wait for text', ['wait', 'text', 'Home'], { + reattemptInfrastructureMiss: () => false, + }), + /step: wait for text/, + ); + assert.equal(calls.filter((args) => args[0] === 'wait').length, 1); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step.txt'))); +}); + +test('a successful step never consults the re-attribution classifier', async () => { + let classified = 0; + const { context, harness, calls } = reissueFixture({ statuses: [0] }); + await harness.runStep(context, 'wait for text', ['wait', 'text', 'Home'], { + reattemptInfrastructureMiss: () => { + classified += 1; + return true; + }, + }); + assert.equal(classified, 0); + assert.equal(calls.filter((args) => args[0] === 'wait').length, 1); +}); + +test('a step with its own allowFailure policy is issued once even when the classifier would retry', async () => { + const { context, harness, calls } = reissueFixture({ statuses: [1] }); + const result = await harness.runStep(context, 'probe', ['is', 'visible', 'id="canary"'], { + allowFailure: true, + reattemptInfrastructureMiss: () => true, + }); + assert.equal(result.status, 1); + assert.equal(calls.filter((args) => args[0] === 'is').length, 1); +}); + +test('a step with expectFailure is issued once even when the classifier would retry', async () => { + const { context, harness, calls } = reissueFixture({ statuses: [1] }); + await harness.runStep(context, 'refusal', ['capabilities', '--x'], { + expectFailure: true, + reattemptInfrastructureMiss: () => true, + }); + assert.equal(calls.filter((args) => args[0] === 'capabilities').length, 1); +}); + test('unwritable artifact output cannot replace the scenario failure', async () => { const { context, harness, reports } = fixture(); fs.renameSync(context.artifactDir, `${context.artifactDir}-moved`); diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index ed1cb7bf92..6ff99abb59 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -65,9 +65,48 @@ type RunStepOptions = { allowFailure?: boolean; commonFlags?: boolean; expectFailure?: boolean; + /** + * A platform's typed classifier for a failed step whose result says the DEVICE prevented the + * observation rather than the product answering wrongly (a wait whose own `details.reason` is + * an observation-prevented verdict). When it answers true, the step is re-issued and only the + * last attempt is asserted, so one runner restart or bridge rebuild costs a repeat of the wait, + * not the job. A step that already owns a miss policy (`allowFailure` / `expectFailure`) never + * gets this layer under it, and a miss the classifier refuses fails exactly as before. + */ + reattemptInfrastructureMiss?: (result: CliJsonResult) => boolean; timeoutMs?: number; }; +/** Issues one step gets when its own miss says infrastructure prevented the observation. */ +const INFRASTRUCTURE_MISS_ISSUES = 2; + +/** + * The re-issue policy a step runs under. A step that already owns a miss policy is issued once: + * its caller reads the failed result or expects failure, and a silent second issue would rewrite + * what that policy observed. + */ +function reattemptPolicyFor( + stepOptions: RunStepOptions, +): ((result: CliJsonResult) => boolean) | undefined { + return stepOptions.allowFailure === true || stepOptions.expectFailure === true + ? undefined + : stepOptions.reattemptInfrastructureMiss; +} + +/** Whether this failed issue of the step is one the platform policy re-issues instead of asserting. */ +function willReissueMiss( + result: CliJsonResult, + reattempt: ((result: CliJsonResult) => boolean) | undefined, + issue: number, +): boolean { + return ( + result.status !== 0 && + reattempt !== undefined && + issue < INFRASTRUCTURE_MISS_ISSUES && + reattempt(result) + ); +} + export function createLiveDeviceContext(options: { artifactRoot: string; session: string; @@ -168,6 +207,32 @@ export function createLiveDeviceHarness< args: string[], stepOptions: RunStepOptions = {}, ): Promise { + const reattempt = reattemptPolicyFor(stepOptions); + for (let issue = 1; ; issue += 1) { + const issuedStep = issue === 1 ? step : `${step} (re-issue after observation-prevented miss)`; + const { result, fullArgs } = await issueStep(context, issuedStep, args, stepOptions); + // A miss the policy will re-issue keeps its step record but writes no failure evidence and + // keeps its error to itself: the evidence and the verdict belong to the final issue, which + // is the only outcome this step is judged on. + if (willReissueMiss(result, reattempt, issue)) continue; + await assertStepOutcome( + context, + issuedStep, + fullArgs, + result, + stepOptions.expectFailure === true && result.status !== 0, + stepOptions, + ); + return result; + } + } + + async function issueStep( + context: Context, + step: string, + args: string[], + stepOptions: RunStepOptions, + ): Promise<{ result: CliJsonResult; fullArgs: string[] }> { const fullArgs = buildStepArgs(context, args, stepOptions); const startedAt = Date.now(); const result = await (options.runCli ?? runBuiltCliJson)(fullArgs, context.env, { @@ -185,9 +250,8 @@ export function createLiveDeviceHarness< status: result.status, step, }); - await assertStepOutcome(context, step, fullArgs, result, failedAsExpected, stepOptions); updateSessionState(context, args[0], result.status); - return result; + return { result, fullArgs }; } function buildStepArgs( From 98321a27900b153cbd7953942e3beec211169848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 8 Oct 2026 23:00:06 +0200 Subject: [PATCH 3/3] test(ios): reject a snapshot quality state outside the declared vocabulary The acquisition assertion only rejected 'sparse' and unexempt 'recovered', so a verdict with any other state (renamed enum, diverged runner) fell through both branches and certified depth facts off a tree the lane never classified. The state set is now pinned to the kernel-declared SNAPSHOT_QUALITY_STATES, with a negative test for states outside it. --- ...s-simulator-e2e-snapshot-acquisition.test.ts | 17 +++++++++++++++++ .../live-snapshot-depth-frontier.ts | 10 +++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts b/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts index d1d632a2d8..d871afe506 100644 --- a/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts +++ b/test/integration/ios-simulator-e2e-snapshot-acquisition.test.ts @@ -74,6 +74,23 @@ test('a sparse capture is rejected whatever strategy it names', () => { ); }); +test('a verdict with a state outside the declared vocabulary is rejected', () => { + // The kernel declares ['healthy', 'recovered', 'sparse'] and the runner's allCases is pinned to + // the same fixture; a verdict outside that set means the two sides diverged, and certifying + // depth facts off a tree the lane cannot classify is the same meaningless-green defect as + // ruling a sparse capture healthy. + for (const state of ['unknown', 'degraded', undefined]) { + assert.throws( + () => + assertSimulatorSnapshotAcquisition( + capture({ ...(state === undefined ? {} : { state }), backend: 'tree' }), + 'regular depth-1 snapshot', + ), + /state outside the declared vocabulary/, + ); + } +}); + test('a capture that declares its hittability evidence missing is rejected', () => { assert.throws( () => diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index bd59d6dcdc..027aa02ec0 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -1,5 +1,7 @@ import assert from 'node:assert/strict'; +import { SNAPSHOT_QUALITY_STATES } from '@agent-device/kernel/snapshot'; + import { assertWaitText, type LiveSnapshotNode as SnapshotNode, @@ -150,7 +152,9 @@ const PRE_SELECTED_REASON_CODES: ReadonlySet = new Set(['deferred', 're * served nothing: `sparse` means no backend served the screen, and `recovered` with a code other * than the two pre-selected ones means the strategy the presented depth semantics belong to failed * mid-capture and another answered (#1569 — two strategies are not comparable views of one screen). - * Either one fails here with the response that proved it. + * Either one fails here with the response that proved it. A verdict whose state is outside the + * kernel-declared `SNAPSHOT_QUALITY_STATES` exhausts nothing and fails too — the lane must not + * certify a tree whose acquisition it does not classify. */ export function assertSimulatorSnapshotAcquisition( result: { json?: any }, @@ -158,6 +162,10 @@ export function assertSimulatorSnapshotAcquisition( ): void { const quality = result.json?.data?.snapshotQuality; if (quality !== undefined) { + assert.ok( + (SNAPSHOT_QUALITY_STATES as readonly unknown[]).includes(quality.state), + `${description} disclosed a quality state outside the declared vocabulary: ${JSON.stringify(result)}`, + ); assert.notEqual( quality.state, 'sparse',