diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index a1b686d072..3594ee6c7a 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -69,6 +69,13 @@ export function runnerSnapshotEntry(nodes: readonly unknown[]): ProviderScenario }; } +// A UI that has gone quiet: every settle capture sees the same tree. How many +// captures the loop spends reaching that verdict is wall-clock, so the count is +// not the contract and is never scripted. +export function quietRunnerSnapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { ...runnerSnapshotEntry(nodes), repeat: true }; +} + export function runnerTapEntry( result: Record, request?: Record, diff --git a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts index c06e936823..0ce303d692 100644 --- a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts +++ b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts @@ -13,6 +13,7 @@ import { RUNNER_NON_HITTABLE_NODES, } from './fixtures.ts'; import { + quietRunnerSnapshotEntry, runnerSnapshotEntry, runnerTapEntry, runnerTapErrorEntry, @@ -188,11 +189,10 @@ test(scenario('settleObservation'), async () => { await withIosContractDaemon( [ // --settle disables the direct path: runtime tree capture, coordinate - // tap, then the settle loop's two stable captures of the changed tree. + // tap, then the settle loop captures the changed tree until it goes quiet. runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerTapEntry({ x: 200, y: 322 }), - runnerSnapshotEntry(RUNNER_CHANGED_NODES), - runnerSnapshotEntry(RUNNER_CHANGED_NODES), + quietRunnerSnapshotEntry(RUNNER_CHANGED_NODES), ], async (daemon, transcript) => { const click = await daemon.callCommand('click', ['label=Continue'], { diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index 435c50344a..7adeeb6b0c 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -130,6 +130,30 @@ function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry }; } +// A UI that has gone quiet: every settle capture sees the same tree. How many +// captures the loop spends reaching that verdict is wall-clock — it polls on a +// 25ms floor and settles once two identical captures span the quiet window — so +// the count is not the contract and is never scripted here. +function quietSnapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { + return { ...snapshotEntry(nodes), repeat: true }; +} + +// A UI that never goes quiet: every capture returns a fresh tree, so the loop +// can never settle no matter how many captures fit in the budget. +function changingSnapshotEntry(nodes: (label: string) => unknown[]): ProviderScenarioProviderEntry { + let capture = 0; + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + repeat: true, + result: () => { + capture += 1; + return { nodes: nodes(`Loading ${capture}`), truncated: false }; + }, + }; +} + function typeEntry(x: number, y: number): ProviderScenarioProviderEntry { return { command: 'ios.runner.type', @@ -155,8 +179,7 @@ test('Provider-backed integration press --settle returns the settled diff and fr // press label=Continue --settle: resolution capture, tap, settle captures snapshotEntry(BEFORE_NODES), tapEntry(200, 322), - snapshotEntry(SETTLED_NODES), - snapshotEntry(SETTLED_NODES), + quietSnapshotEntry(SETTLED_NODES), // press @e2 (the Done ref from the settled diff): tap on the stored tree tapEntry(200, 522), ]); @@ -212,7 +235,14 @@ test('Provider-backed integration press --settle returns the settled diff and fr }; assert.ok(settle, 'press --settle must return a settle observation'); assert.equal(settle.settled, true); - assert.equal(settle.captures, 2); + // Snapshot-floor economy: a quiet UI settles on the two identical + // captures the verdict needs, or three when the second lands a hair short + // of the quiet window. Anything above that is a regression in what settle + // spends; pinning a single number would assert the runner's speed. + assert.ok( + settle.captures >= 2 && settle.captures <= 3, + `settle should cost 2-3 captures, got ${settle.captures}`, + ); assert.equal(typeof settle.refsGeneration, 'number'); assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 2, unchanged: 1 }); const added = settle.diff?.lines.find((line) => line.kind === 'added'); @@ -229,11 +259,19 @@ test('Provider-backed integration press --settle returns the settled diff and fr // directly on the stored settled tree with no fresh snapshot round trip // and no stale-refs warning. The plain `@e2` would require a complete // frame. + const callsBeforeFollowUp = runnerTranscript.calls.length; const followUp = await daemon.callCommand('press', [`@e2~s${settle.refsGeneration}`], {}); const followUpData = assertRpcOk(followUp); assert.equal(followUpData.warning, undefined); assert.equal(followUpData.x, 200); assert.equal(followUpData.y, 522); + // The round trip settle exists to remove: the follow-up spends a tap and + // nothing else. Asserted directly, since a quiet-UI entry would happily + // serve a stray capture. + assert.deepEqual( + runnerTranscript.calls.slice(callsBeforeFollowUp).map((call) => call.command), + ['ios.runner.tap'], + ); runnerTranscript.assertComplete(); }, @@ -258,13 +296,11 @@ test('Provider-backed integration never-settled press --settle does not issue di }, ]; const runnerTranscript = createProviderTranscript([ - // press label=Continue --settle: resolution capture, tap, then a changing - // settle stream. The loop times out; the final capture is not actionable. + // press label=Continue --settle: resolution capture, tap, then a settle + // stream that never repeats itself, so the loop can only ever time out. snapshotEntry(BEFORE_NODES), tapEntry(200, 322), - snapshotEntry(loadingNodes('Loading 1')), - snapshotEntry(loadingNodes('Loading 2')), - snapshotEntry(SETTLED_NODES), + changingSnapshotEntry(loadingNodes), ]); const appleRunnerProvider = createAppleRunnerProviderFromTranscript( runnerTranscript, @@ -293,7 +329,10 @@ test('Provider-backed integration never-settled press --settle does not issue di const press = await daemon.callCommand('press', ['label=Continue'], { settle: true, settleQuietMs: 25, - timeoutMs: 60, + // Budget the loop spends without settling. Wide enough that a loaded + // runner's capture cannot outlast it — a capture that overruns the + // budget reports the stalled hint instead of this one. + timeoutMs: 300, }); const pressData = assertRpcOk(press); const settle = pressData.settle as { @@ -324,8 +363,7 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch // no added refs, so the tail is the only actionable-target payload. snapshotEntry(MODAL_BEFORE_NODES), tapEntry(200, 422), - snapshotEntry(MODAL_DISMISSED_NODES), - snapshotEntry(MODAL_DISMISSED_NODES), + quietSnapshotEntry(MODAL_DISMISSED_NODES), // press @e3 (the Continue ref from the tail): tap on the stored tree. tapEntry(200, 322), ]); @@ -392,11 +430,16 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch // The tail's ref acts directly on the stored settled tree, same as an // added-line ref would — consumed in pinned form from the partial frame // (ADR 0014). + const callsBeforeFollowUp = runnerTranscript.calls.length; const followUp = await daemon.callCommand('press', [`@e3~s${settle.refsGeneration}`], {}); const followUpData = assertRpcOk(followUp); assert.equal(followUpData.warning, undefined); assert.equal(followUpData.x, 200); assert.equal(followUpData.y, 322); + assert.deepEqual( + runnerTranscript.calls.slice(callsBeforeFollowUp).map((call) => call.command), + ['ios.runner.tap'], + ); runnerTranscript.assertComplete(); }, @@ -651,8 +694,7 @@ test('Provider-backed integration fill --settle summoning the keyboard still att // fresh capture), the runner types, then the settle loop captures. The // keyboard window appears and the field re-labels itself. typeEntry(201, 149), - snapshotEntry(FILL_SETTLED_NODES), - snapshotEntry(FILL_SETTLED_NODES), + quietSnapshotEntry(FILL_SETTLED_NODES), // press @e17 (the "Discard and go back" ref from the tail): tap on the // stored settled tree. tapEntry(201, 202), @@ -724,11 +766,16 @@ test('Provider-backed integration fill --settle summoning the keyboard still att assert.equal(settle.tailTruncated, undefined); assert.equal(typeof settle.refsGeneration, 'number'); + const callsBeforeFollowUp = runnerTranscript.calls.length; const followUp = await daemon.callCommand('press', [`@e17~s${settle.refsGeneration}`], {}); const followUpData = assertRpcOk(followUp); assert.equal(followUpData.warning, undefined); assert.equal(followUpData.x, 201); assert.equal(followUpData.y, 202); + assert.deepEqual( + runnerTranscript.calls.slice(callsBeforeFollowUp).map((call) => call.command), + ['ios.runner.tap'], + ); runnerTranscript.assertComplete(); }, diff --git a/test/integration/provider-scenarios/transcript.test.ts b/test/integration/provider-scenarios/transcript.test.ts index b7282189e6..df33112f30 100644 --- a/test/integration/provider-scenarios/transcript.test.ts +++ b/test/integration/provider-scenarios/transcript.test.ts @@ -15,6 +15,93 @@ test('provider transcript matches expected calls without requiring incidental or transcript.assertComplete(); }); +test('repeat entries serve every matching call and never count as unconsumed', () => { + const transcript = createProviderTranscript([ + { command: 'ios.runner.snapshot', repeat: true, result: { ok: 'snapshot' } }, + { command: 'ios.runner.tap', result: { ok: 'tap' } }, + ]); + + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'snapshot' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'snapshot' }); + assert.deepEqual(transcript.next('ios.runner.tap'), { ok: 'tap' }); + + // The repeat entry stays pending forever; only the one-shot tap had to land. + transcript.assertComplete(); + assert.equal(transcript.calls.length, 3); +}); + +test('one-shot entries still outrank a repeat entry and remain required', () => { + const transcript = createProviderTranscript([ + { command: 'ios.runner.snapshot', result: { ok: 'first' } }, + { command: 'ios.runner.snapshot', repeat: true, result: { ok: 'rest' } }, + ]); + + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'first' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'rest' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'rest' }); + transcript.assertComplete(); +}); + +test('a repeat declared before a matching one-shot does not shadow it', () => { + // Outranking is a rule, not an accident of declaration order: taking the + // first match would serve the repeat forever and strand the one-shot. + const transcript = createProviderTranscript([ + { command: 'ios.runner.snapshot', repeat: true, result: { ok: 'rest' } }, + { command: 'ios.runner.snapshot', result: { ok: 'first' } }, + ]); + + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'first' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'rest' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { ok: 'rest' }); + transcript.assertComplete(); +}); + +test('ordered transcripts reject repeat entries outright', () => { + // Ordered lookup only ever reads entry 0, so a repeat there never advances + // and makes every later entry unreachable. Refuse the combination instead of + // failing later as a confusing command mismatch. + assert.throws( + () => + createOrderedProviderTranscript([ + { command: 'ios.runner.snapshot', repeat: true, result: { ok: 'snapshot' } }, + { command: 'ios.runner.tap', result: { ok: 'tap' } }, + ]), + /repeat/i, + ); +}); + +test('unconsumed one-shot entries still fail assertComplete alongside a repeat entry', () => { + const transcript = createProviderTranscript([ + { command: 'ios.runner.snapshot', repeat: true, result: { ok: 'snapshot' } }, + { command: 'ios.runner.tap', result: { ok: 'tap' } }, + ]); + + transcript.next('ios.runner.snapshot'); + assert.throws(() => transcript.assertComplete(), /Unconsumed provider transcript entries.*tap/s); +}); + +test('a result factory is invoked per call, so repeated calls can return fresh results', () => { + let capture = 0; + const transcript = createProviderTranscript([ + { + command: 'ios.runner.snapshot', + repeat: true, + result: () => { + capture += 1; + return { label: `Loading ${capture}` }; + }, + }, + ]); + + assert.deepEqual(transcript.next('ios.runner.snapshot'), { label: 'Loading 1' }); + assert.deepEqual(transcript.next('ios.runner.snapshot'), { label: 'Loading 2' }); + // The recorded call carries the resolved result, not the factory. + assert.deepEqual( + transcript.calls.map((call) => call.result), + [{ label: 'Loading 1' }, { label: 'Loading 2' }], + ); +}); + test('ordered provider transcript remains available when ordering is the contract', () => { const transcript = createOrderedProviderTranscript([ { command: 'ios.runner.snapshot', result: { ok: 'snapshot' } }, diff --git a/test/integration/provider-scenarios/transcript.ts b/test/integration/provider-scenarios/transcript.ts index 7af3579f69..00313ac141 100644 --- a/test/integration/provider-scenarios/transcript.ts +++ b/test/integration/provider-scenarios/transcript.ts @@ -12,7 +12,21 @@ export interface ProviderScenarioProviderEntry< > extends ProviderScenarioProviderScope { command: string; request?: unknown; - result?: TResult; + /** A factory is invoked per call, so repeated calls can return fresh results. */ + result?: TResult | (() => TResult); + /** + * Serves every matching call instead of being consumed by the first, and never + * counts as unconsumed. Use it where the call COUNT is not the contract — a + * quiet UI returns the same tree to every capture, a busy one returns a fresh + * tree per capture (pair with a `result` factory). Scripting an exact count + * there would assert the runner's speed rather than the behaviour. + * + * A repeat is the FALLBACK for its command: matching one-shot entries are + * always served first, whatever order the entries are declared in, so a + * repeat can never strand one. Unsupported in ordered transcripts, where a + * repeat would never advance the queue. + */ + repeat?: boolean; error?: Error | string; } @@ -39,6 +53,11 @@ export function createProviderTranscript( entries: readonly ProviderScenarioProviderEntry[], options: { ordered?: boolean } = {}, ): ProviderScenarioTranscript { + if (options.ordered && entries.some((entry) => entry.repeat)) { + throw new Error( + 'Ordered provider transcripts cannot use `repeat` entries: ordered lookup only reads the head, so a repeat never advances and strands every entry behind it.', + ); + } const pending = [...entries]; const calls: ProviderScenarioProviderCall[] = []; @@ -54,25 +73,23 @@ export function createProviderTranscript( request?: unknown, scope: ProviderScenarioProviderScope = {}, ): TResult { - const entryIndex = options.ordered - ? 0 - : pending.findIndex((candidate) => - providerEntryMatches(candidate, command, request, scope), - ); - const entry = entryIndex >= 0 ? pending.splice(entryIndex, 1)[0] : undefined; + const entryIndex = options.ordered ? 0 : findEntryIndex(pending, command, request, scope); + const entry = entryIndex >= 0 ? pending[entryIndex] : undefined; assert.ok(entry, `Unexpected provider call: ${formatCall(command, scope)}`); + if (!entry.repeat) pending.splice(entryIndex, 1); assert.equal(command, entry.command, 'Provider command mismatch'); assertScope(scope, entry); if (Object.hasOwn(entry, 'request')) { assert.deepEqual(request, entry.request, 'Provider request mismatch'); } + const result = resolveEntryResult(entry) as TResult; const call = { command, request, deviceId: scope.deviceId, platform: scope.platform, - result: entry.result as TResult, + result, }; calls.push(call); @@ -80,13 +97,14 @@ export function createProviderTranscript( throw entry.error instanceof Error ? entry.error : new Error(entry.error); } - return entry.result as TResult; + return result; }, assertComplete() { + const outstanding = pending.filter((entry) => !entry.repeat); assert.equal( - pending.length, + outstanding.length, 0, - `Unconsumed provider transcript entries: ${pending.map(formatEntry).join(', ')}`, + `Unconsumed provider transcript entries: ${outstanding.map(formatEntry).join(', ')}`, ); }, }; @@ -98,6 +116,27 @@ export function createOrderedProviderTranscript( return createProviderTranscript(entries, { ordered: true }); } +/** + * One-shot entries are served before repeats regardless of declaration order: a + * repeat is its command's fallback, so taking the first match would let one + * shadow a one-shot forever and strand it as permanently unconsumed. + */ +function findEntryIndex( + pending: readonly ProviderScenarioProviderEntry[], + command: string, + request: unknown, + scope: ProviderScenarioProviderScope, +): number { + const matches = (candidate: ProviderScenarioProviderEntry): boolean => + providerEntryMatches(candidate, command, request, scope); + const oneShotIndex = pending.findIndex((candidate) => !candidate.repeat && matches(candidate)); + return oneShotIndex >= 0 ? oneShotIndex : pending.findIndex(matches); +} + +function resolveEntryResult(entry: ProviderScenarioProviderEntry): unknown { + return typeof entry.result === 'function' ? (entry.result as () => unknown)() : entry.result; +} + function providerEntryMatches( entry: ProviderScenarioProviderEntry, command: string,