diff --git a/src/commands/interaction/runtime/interactions.test.ts b/src/commands/interaction/runtime/interactions.test.ts index 4e35d61a7..61a337900 100644 --- a/src/commands/interaction/runtime/interactions.test.ts +++ b/src/commands/interaction/runtime/interactions.test.ts @@ -127,7 +127,10 @@ test('runtime fill uses backend ref primitive without resolving snapshot geometr assert.deepEqual(calls, [{ ref: '@e1', text: 'hello', delayMs: 25 }]); assert.equal(result.kind, 'ref'); assert.equal(result.point, undefined); - assert.equal(result.node, undefined); + // ADR 0012 decision 3: the preflight's guard lookup supplies the + // record-time evidence node on the runtime result. + assert.equal(result.node?.ref, 'e1'); + assert.ok(Array.isArray(result.preActionNodes)); assert.deepEqual(result.target, { kind: 'ref', ref: '@e1' }); assert.equal(result.text, 'hello'); assert.deepEqual(result.backendResult, { ref: 'e1', text: 'hello' }); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index f2b4c4441..797c9113a 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -492,7 +492,12 @@ export async function preflightNativeRefInteraction( options: CommandContext, target: Extract, action: InteractionAction, -): Promise<{ targetHittable?: boolean; hint?: string }> { +): Promise<{ + targetHittable?: boolean; + hint?: string; + node?: SnapshotNode; + preActionNodes?: SnapshotNode[]; +}> { const session = await runtime.sessions.get(options.session ?? 'default'); const nodes = session?.snapshot?.nodes; if (!nodes || normalizeRef(target.ref) === null) return {}; @@ -502,7 +507,13 @@ export async function preflightNativeRefInteraction( if (!resolved) return {}; assertInteractionNotBlocked(resolved.node, `Ref ${target.ref}`, action); assertVisibleRefTarget(resolved.node, nodes, target.ref, action); - return describeNonHittableTarget(resolved.node, action); + return { + ...describeNonHittableTarget(resolved.node, action), + // ADR 0012 decision 3: the guard lookup above doubles as the record-time + // evidence source for the fast path, at zero extra capture cost. + node: resolved.node, + preActionNodes: nodes, + }; } // isNodeVisibleOnScreen (not the effective-viewport form): items inside an diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 814f30970..3ec4ee563 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -78,12 +78,16 @@ export type GetCommandResult = text: string; node: SnapshotNode; selectorChain?: string[]; + /** ADR 0012 decision 3: the tree `node` was resolved from, for record-time evidence. */ + preActionNodes: SnapshotNode[]; } | { kind: 'attrs'; target: ResolvedTarget; node: SnapshotNode; selectorChain?: string[]; + /** ADR 0012 decision 3: the tree `node` was resolved from, for record-time evidence. */ + preActionNodes: SnapshotNode[]; }; export type GetTextCommandOptions = CommandContext & @@ -199,11 +203,12 @@ export const getCommand: RuntimeCommand = a action: 'get', }); const target = { kind: 'ref' as const, ref: `@${resolved.ref}` }; + const preActionNodes = capture.snapshot.nodes; if (options.property === 'attrs') { - return { kind: 'attrs', target, node: resolved.node, selectorChain }; + return { kind: 'attrs', target, node: resolved.node, selectorChain, preActionNodes }; } const text = await readText(runtime, capture, resolved.node); - return { kind: 'text', target, text, node: resolved.node, selectorChain }; + return { kind: 'text', target, text, node: resolved.node, selectorChain, preActionNodes }; } const resolved = await resolveSelectorNode(runtime, options, options.session ?? 'default', { @@ -221,6 +226,7 @@ export const getCommand: RuntimeCommand = a target: { kind: 'selector', selector: resolved.selector }, node: resolved.node, selectorChain, + preActionNodes: resolved.capture.snapshot.nodes, }; } @@ -231,6 +237,7 @@ export const getCommand: RuntimeCommand = a text, node: resolved.node, selectorChain, + preActionNodes: resolved.capture.snapshot.nodes, }; }; diff --git a/src/daemon/__tests__/session-store.test.ts b/src/daemon/__tests__/session-store.test.ts index 622e8600b..6318c4fdb 100644 --- a/src/daemon/__tests__/session-store.test.ts +++ b/src/daemon/__tests__/session-store.test.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import { SessionStore } from '../session-store.ts'; import type { SessionState } from '../types.ts'; import { buildRequestFinishedEvent } from '../session-event-log.ts'; +import type { TargetAnnotationV1 } from '../../replay/target-identity.ts'; type RecordActionEntry = Parameters[1]; @@ -592,3 +593,56 @@ test('writeSessionLog preserves significant whitespace and empty string argument /--metro-host " host\\t" --launch-url "myapp:\/\/dev "/, ]); }); + +// --------------------------------------------------------------------------- +// ADR 0012 decision 3: recorded target-v1 evidence flows end to end through +// `recordAction` → `SessionScriptWriter` into the `.ad` file, immediately +// before the action it annotates. +// --------------------------------------------------------------------------- + +const SAVE_TARGET_EVIDENCE: TargetAnnotationV1 = { + id: 'save', + role: 'button', + label: 'Save', + ancestry: [{ role: 'toolbar', label: 'Editor' }], + sibling: 0, + viewportOrder: 0, + verification: 'verified', +}; + +test('writeSessionLog emits the target-v1 annotation immediately before its action line', () => { + const fixture = makeFixture('agent-device-session-log-target-evidence-'); + recordOpen(fixture.store, fixture.session); + fixture.store.recordAction(fixture.session, { + command: 'click', + positionals: ['@e12'], + flags: { platform: 'ios' }, + result: {}, + targetEvidence: SAVE_TARGET_EVIDENCE, + }); + recordClose(fixture.store, fixture.session); + + const script = writeScript(fixture); + const lines = script.trim().split('\n'); + const clickLineIndex = lines.findIndex((line) => line.startsWith('click ')); + assert.ok(clickLineIndex > 0); + assert.equal( + lines[clickLineIndex - 1], + '# agent-device:target-v1 {"id":"save","role":"button","label":"Save","ancestry":[{"role":"toolbar","label":"Editor"}],"sibling":0,"viewportOrder":0,"verification":"verified"}', + ); +}); + +test('writeSessionLog never fabricates a target-v1 annotation for actions recorded without evidence', () => { + const fixture = makeFixture('agent-device-session-log-no-target-evidence-'); + recordOpen(fixture.store, fixture.session); + fixture.store.recordAction(fixture.session, { + command: 'click', + positionals: ['@e12'], + flags: { platform: 'ios' }, + result: {}, + }); + recordClose(fixture.store, fixture.session); + + const script = writeScript(fixture); + assert.equal(/agent-device:target-v1/.test(script), false); +}); diff --git a/src/daemon/__tests__/session-target-evidence.test.ts b/src/daemon/__tests__/session-target-evidence.test.ts new file mode 100644 index 000000000..81d278db7 --- /dev/null +++ b/src/daemon/__tests__/session-target-evidence.test.ts @@ -0,0 +1,468 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { RawSnapshotNode, SnapshotNode } from '../../kernel/snapshot.ts'; +import { computeTargetEvidence } from '../session-target-evidence.ts'; +import { + parseTargetAnnotationV1Payload, + serializeTargetAnnotationV1, + utf8ByteLength, + TARGET_ANNOTATION_MAX_ANCESTRY, + TARGET_ANNOTATION_MAX_FIELD_BYTES, + TARGET_ANNOTATION_MAX_PAYLOAD_BYTES, +} from '../../replay/target-identity.ts'; + +function toSnapshotNodes(raw: RawSnapshotNode[]): SnapshotNode[] { + return raw.map((node, position) => ({ ...node, ref: `e${position + 1}` })); +} + +function findByLabel(nodes: SnapshotNode[], label: string): SnapshotNode { + const found = nodes.find((node) => node.label === label); + if (!found) throw new Error(`fixture missing node with label ${label}`); + return found; +} + +// --------------------------------------------------------------------------- +// Basic record-time write algorithm (decision 3 steps 1-5) +// --------------------------------------------------------------------------- + +function toolbarFixture(): SnapshotNode[] { + return toSnapshotNodes([ + { index: 0, type: 'Window', depth: 0 }, + { index: 1, type: 'Toolbar', label: 'Editor', depth: 1, parentIndex: 0 }, + { + index: 2, + type: 'Button', + identifier: 'save', + label: 'Save', + rect: { x: 10, y: 10, width: 40, height: 20 }, + depth: 2, + parentIndex: 1, + }, + ]); +} + +test('computeTargetEvidence: single unambiguous node is verified with the expected identity/ancestry/rect', () => { + const nodes = toolbarFixture(); + const winner = findByLabel(nodes, 'Save'); + const evidence = computeTargetEvidence({ node: winner, preActionNodes: nodes }); + assert.ok(evidence); + assert.deepEqual(evidence, { + id: 'save', + role: 'button', + label: 'Save', + ancestry: [{ role: 'toolbar', label: 'Editor' }, { role: 'window' }], + sibling: 0, + viewportOrder: 0, + rect: { x: 10, y: 10, width: 40, height: 20 }, + verification: 'verified', + }); +}); + +// --------------------------------------------------------------------------- +// Scroll region + viewport order +// --------------------------------------------------------------------------- + +function scrollableListFixture(): SnapshotNode[] { + return toSnapshotNodes([ + { index: 0, type: 'Window', depth: 0 }, + { + index: 1, + type: 'ScrollView', + identifier: 'editor-scroll', + depth: 1, + parentIndex: 0, + }, + { + index: 2, + type: 'Cell', + label: 'Row', + rect: { x: 0, y: 100, width: 100, height: 20 }, + depth: 2, + parentIndex: 1, + }, + { + index: 3, + type: 'Cell', + label: 'Row', + rect: { x: 0, y: 50, width: 100, height: 20 }, + depth: 2, + parentIndex: 1, + }, + { + index: 4, + type: 'Cell', + label: 'Row', + rect: { x: 0, y: 0, width: 100, height: 20 }, + depth: 2, + parentIndex: 1, + }, + ]); +} + +test('computeTargetEvidence: scrollRegion is the nearest scrollable ancestor local identity, viewportOrder is top-to-bottom within it', () => { + const nodes = scrollableListFixture(); + const winner = nodes[2]!; // rect y:100 -> should be last (index 2) in top-to-bottom order + const evidence = computeTargetEvidence({ node: winner, preActionNodes: nodes }); + assert.ok(evidence); + assert.deepEqual(evidence.scrollRegion, { role: 'scrollview', id: 'editor-scroll' }); + // rows are at y=0,50,100 -> the y:100 row is ordinal 2 (0-based) top-to-bottom + assert.equal(evidence.viewportOrder, 2); + assert.equal(evidence.verification, 'verified'); +}); + +test('computeTargetEvidence: a stable scroll-region ID takes precedence over a changed label', () => { + const recordedNodes = scrollableListFixture(); + recordedNodes[1]!.label = 'Inbox'; + const recorded = computeTargetEvidence({ + node: recordedNodes[2]!, + preActionNodes: recordedNodes, + }); + + const currentNodes = scrollableListFixture(); + currentNodes[1]!.label = 'Messages'; + const current = computeTargetEvidence({ node: currentNodes[2]!, preActionNodes: currentNodes }); + + assert.ok(recorded); + assert.ok(current); + assert.deepEqual(recorded.scrollRegion, { role: 'scrollview', id: 'editor-scroll' }); + assert.deepEqual(current.scrollRegion, recorded.scrollRegion); +}); + +// --------------------------------------------------------------------------- +// Duplicate identity resolved by sibling / viewportOrder, still verified +// (decision 3's self-check succeeds by construction whenever the capture +// supplies the needed structural data). +// --------------------------------------------------------------------------- + +test('computeTargetEvidence: duplicate identity across two parents is still verified via the sibling ordinal', () => { + const nodes = toSnapshotNodes([ + { index: 0, type: 'Window', depth: 0 }, + { index: 1, type: 'Row', depth: 1, parentIndex: 0 }, + { + index: 2, + type: 'Button', + label: 'Go back', + rect: { x: 0, y: 0, width: 40, height: 20 }, + depth: 2, + parentIndex: 1, + }, + { index: 3, type: 'Row', depth: 1, parentIndex: 0 }, + { + index: 4, + type: 'Button', + label: 'Go back', + rect: { x: 0, y: 200, width: 40, height: 20 }, + depth: 2, + parentIndex: 3, + }, + ]); + const winner = nodes[4]!; // the second "Go back" button, under the second Row + const evidence = computeTargetEvidence({ node: winner, preActionNodes: nodes }); + assert.ok(evidence); + assert.equal(evidence.id, undefined); + assert.equal(evidence.role, 'button'); + assert.equal(evidence.label, 'Go back'); + // Both buttons are child index 0 of their own (identical, unlabeled) Row — + // ancestry alone does not disambiguate them, and sibling is identical (0) + // for both. Only document-order-based viewportOrder within the (no + // scroll-region) partition can isolate the winner. + assert.equal(evidence.sibling, 0); + assert.equal(evidence.verification, 'verified'); +}); + +// --------------------------------------------------------------------------- +// Writer-parser invariant: max-size (256-byte) labels x K=8 ancestry reduces +// rather than rejecting its own output. +// --------------------------------------------------------------------------- + +test('computeTargetEvidence: max-size labels across K=8 ancestors reduce ancestry to fit 4 KiB, and the writer never rejects its own output', () => { + const maxLabel = 'x'.repeat(TARGET_ANNOTATION_MAX_FIELD_BYTES); + const raw: RawSnapshotNode[] = []; + // Build a deep chain of 9 ancestors (root .. depth 8) with BOTH role and + // label maxed at 256 bytes, plus the winning leaf at depth 9. 8 such + // entries alone exceed the 4 KiB payload cap, forcing root-side reduction. + for (let depth = 0; depth < 9; depth += 1) { + raw.push({ + index: depth, + type: maxLabel, + label: maxLabel, + depth, + parentIndex: depth > 0 ? depth - 1 : undefined, + }); + } + raw.push({ + index: 9, + type: 'Button', + label: maxLabel, + identifier: maxLabel, + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 9, + parentIndex: 8, + }); + const nodes = toSnapshotNodes(raw); + const winner = nodes.at(-1)!; + + const evidence = computeTargetEvidence({ node: winner, preActionNodes: nodes }); + assert.ok(evidence); + assert.ok(evidence.ancestry.length <= TARGET_ANNOTATION_MAX_ANCESTRY); + assert.ok( + evidence.ancestry.length < TARGET_ANNOTATION_MAX_ANCESTRY, + 'ancestry must have been reduced', + ); + // Nearest-ancestor-first: the first (kept) entries are the leaf-side ones. + assert.equal(evidence.ancestry[0]?.label, maxLabel); + + const json = serializeTargetAnnotationV1(evidence); + assert.ok(utf8ByteLength(json) <= TARGET_ANNOTATION_MAX_PAYLOAD_BYTES); + // The writer-parser invariant: the writer's own output must never be + // rejected by its own parser. + const reparsed = parseTargetAnnotationV1Payload(json); + assert.deepEqual(reparsed, evidence); +}); + +test('computeTargetEvidence: a root node with no ancestors has empty ancestry and is still verified', () => { + const nodes = toSnapshotNodes([ + { + index: 0, + type: 'Button', + identifier: 'root-button', + label: 'Root', + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 0, + }, + ]); + const evidence = computeTargetEvidence({ node: nodes[0]!, preActionNodes: nodes }); + assert.ok(evidence); + assert.deepEqual(evidence.ancestry, []); + assert.equal(evidence.verification, 'verified'); +}); + +// --------------------------------------------------------------------------- +// Fixture realism: a real-capture-shaped tree (react-navigation-style bottom +// tab bar) with undefined/false `hittable` and anonymous wrapper nodes — see +// test/integration/provider-scenarios/ios-world.ts (PR #1172 pattern). +// --------------------------------------------------------------------------- + +function bottomTabsRealCaptureFixture(): SnapshotNode[] { + return toSnapshotNodes([ + { + index: 0, + type: 'Application', + label: 'React Navigation Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + enabled: true, + hittable: true, + depth: 0, + }, + { + index: 1, + type: 'Window', + rect: { x: 0, y: 0, width: 402, height: 874 }, + enabled: true, + hittable: true, + depth: 1, + parentIndex: 0, + }, + { + index: 2, + type: 'ScrollView', + label: 'Contacts', + rect: { x: 0, y: 116, width: 402, height: 675 }, + enabled: true, + hittable: false, + hiddenContentBelow: true, + depth: 2, + parentIndex: 1, + }, + { + index: 3, + type: 'StaticText', + label: 'Marissa Castillo', + rect: { x: 52, y: 132, width: 110, height: 17 }, + enabled: true, + // Real captures commonly omit `hittable` entirely on plain text nodes + // rather than reporting it `false` — anonymous, undefined-hittable. + depth: 3, + parentIndex: 2, + }, + { + index: 4, + // Anonymous wrapper: no identifier/label of its own — a real shape + // this ADR's ancestry entries must tolerate (`role` present, `label` + // omitted). + type: 'Other', + rect: { x: 0, y: 791, width: 402, height: 83 }, + enabled: true, + hittable: false, + depth: 2, + parentIndex: 1, + }, + { + index: 5, + type: 'Button', + label: 'Article, unselected', + identifier: 'article', + rect: { x: 0, y: 791, width: 101, height: 49 }, + enabled: true, + hittable: false, + depth: 3, + parentIndex: 4, + }, + { + index: 6, + type: 'Button', + label: 'Chat, unselected', + identifier: 'chat', + rect: { x: 101, y: 791, width: 100, height: 49 }, + enabled: true, + hittable: false, + depth: 3, + parentIndex: 4, + }, + ]); +} + +test('computeTargetEvidence: real-capture-shaped tree (undefined hittable, anonymous wrapper ancestor)', () => { + const nodes = bottomTabsRealCaptureFixture(); + const winner = findByLabel(nodes, 'Article, unselected'); + const evidence = computeTargetEvidence({ node: winner, preActionNodes: nodes }); + assert.ok(evidence); + assert.equal(evidence.id, 'article'); + assert.equal(evidence.role, 'button'); + // The anonymous "Other" wrapper ancestor carries a role but no label. + assert.deepEqual(evidence.ancestry[0], { role: 'other' }); + assert.equal(evidence.ancestry[1]?.role, 'window'); + assert.equal(evidence.ancestry[2]?.role, 'application'); + assert.equal(evidence.scrollRegion, undefined); // tab bar sits outside the ScrollView + assert.equal(evidence.sibling, 0); // first child of the "Other" tab-bar row + assert.equal(evidence.verification, 'verified'); + + const json = serializeTargetAnnotationV1(evidence); + assert.deepEqual(parseTargetAnnotationV1Payload(json), evidence); +}); + +// --------------------------------------------------------------------------- +// Self-consistency: a node whose own id/label exceeds the 256-byte field cap +// must still match ITSELF during the identity-set scan. The recorded +// identity is truncated (writer-parser invariant); comparing an untruncated +// candidate against it would spuriously exclude the winner from its own +// identity set and produce a false 'unverifiable'. +// --------------------------------------------------------------------------- + +test('computeTargetEvidence: a node with an over-cap id still matches itself and is verified', () => { + const overCapId = 'save-'.repeat(100); // 500 bytes, well over the 256-byte field cap + const nodes = toSnapshotNodes([ + { + index: 0, + type: 'Button', + identifier: overCapId, + label: 'Save', + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 0, + }, + ]); + const evidence = computeTargetEvidence({ node: nodes[0]!, preActionNodes: nodes }); + assert.ok(evidence); + assert.equal(evidence.id, overCapId.slice(0, TARGET_ANNOTATION_MAX_FIELD_BYTES)); + assert.equal(evidence.verification, 'verified'); +}); + +// --------------------------------------------------------------------------- +// Worst-case verification sizing: "unverifiable" serializes 4 bytes longer +// than "verified". The reduction loop must size each candidate against the +// worst-case value so a fail-closed self-check downgrade can never push an +// already-accepted payload over the 4 KiB cap. The window is only 4 bytes +// wide, so sweep a tunable root-side label length across the boundary — some +// length in the sweep necessarily lands inside the window, where a +// placeholder-sized ("verified") check would accept a payload whose +// downgraded form overflows. +// --------------------------------------------------------------------------- + +test('computeTargetEvidence: reduction sizes against the worst-case verification value across the 4 KiB boundary', () => { + const maxField = 'x'.repeat(TARGET_ANNOTATION_MAX_FIELD_BYTES); + + const buildChain = (tunableLabelLength: number): SnapshotNode[] => { + const raw: RawSnapshotNode[] = []; + // Root (index 0) .. leaf's parent (index 7): 8 ancestors total (K=8). + // Root-side entries are dropped first, so the tunable label sits at the + // root to sweep total payload size in 1-byte steps. + raw.push({ index: 0, type: 'Window', label: 'w'.repeat(tunableLabelLength), depth: 0 }); + raw.push({ index: 1, type: 'View', label: 'v', depth: 1, parentIndex: 0 }); + for (let depth = 2; depth < 8; depth += 1) { + raw.push({ index: depth, type: maxField, label: maxField, depth, parentIndex: depth - 1 }); + } + raw.push({ + index: 8, + type: maxField, + label: maxField, + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 8, + parentIndex: 7, + }); + return toSnapshotNodes(raw); + }; + + let sawFullAncestry = false; + let sawReducedAncestry = false; + for (let tunable = 0; tunable <= TARGET_ANNOTATION_MAX_FIELD_BYTES; tunable += 1) { + const nodes = buildChain(tunable); + const evidence = computeTargetEvidence({ node: nodes.at(-1)!, preActionNodes: nodes }); + assert.ok(evidence); + if (evidence.ancestry.length === TARGET_ANNOTATION_MAX_ANCESTRY) sawFullAncestry = true; + else sawReducedAncestry = true; + // The emitted payload must fit even re-serialized with the longer + // verification value — the invariant the worst-case sizing guarantees. + const worstCase = serializeTargetAnnotationV1({ ...evidence, verification: 'unverifiable' }); + assert.ok( + utf8ByteLength(worstCase) <= TARGET_ANNOTATION_MAX_PAYLOAD_BYTES, + `worst-case payload overflows at tunable label length ${tunable}`, + ); + // And the parser accepts the writer's actual output, as always. + parseTargetAnnotationV1Payload(serializeTargetAnnotationV1(evidence)); + } + // The sweep must actually cross the reduction boundary for the window to + // have been exercised. + assert.ok(sawFullAncestry, 'sweep never produced an unreduced payload — fixture too large'); + assert.ok(sawReducedAncestry, 'sweep never forced a reduction — fixture too small'); +}); + +// --------------------------------------------------------------------------- +// Broken parent linkage is a capture anomaly (decision 3): fail closed to +// 'unverifiable', never 'verified' on structural data that cannot be trusted. +// --------------------------------------------------------------------------- + +test('computeTargetEvidence: a dangling parentIndex records unverifiable, never verified', () => { + const nodes = toSnapshotNodes([ + { + index: 0, + type: 'Button', + identifier: 'save', + label: 'Save', + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 1, + parentIndex: 42, // no node with index 42 exists + }, + ]); + const evidence = computeTargetEvidence({ node: nodes[0]!, preActionNodes: nodes }); + assert.ok(evidence); + assert.equal(evidence.verification, 'unverifiable'); +}); + +test('computeTargetEvidence: a parent cycle records unverifiable, never verified', () => { + const nodes = toSnapshotNodes([ + { index: 0, type: 'View', depth: 0, parentIndex: 1 }, + { index: 1, type: 'View', depth: 1, parentIndex: 0 }, + { + index: 2, + type: 'Button', + identifier: 'save', + label: 'Save', + rect: { x: 0, y: 0, width: 10, height: 10 }, + depth: 2, + parentIndex: 1, + }, + ]); + const evidence = computeTargetEvidence({ node: nodes[2]!, preActionNodes: nodes }); + assert.ok(evidence); + assert.equal(evidence.verification, 'unverifiable'); +}); diff --git a/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts new file mode 100644 index 000000000..23e40be42 --- /dev/null +++ b/src/daemon/handlers/__tests__/interaction-target-evidence.test.ts @@ -0,0 +1,231 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { handleInteractionCommands } from '../interaction.ts'; +import { attachRefs, type RawSnapshotNode } from '../../../kernel/snapshot.ts'; +import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; +import type { CommandFlags } from '../../../core/dispatch.ts'; +import type { SessionState } from '../../types.ts'; + +// ADR 0012 decision 3, daemon-routed recording: target-v1 evidence is +// computed only while the session is being recorded, lands on the recorded +// action (never the public response), and the raw resolved node/tree reaches +// neither the client nor session history. Covers the touch path +// (finalizeTouchInteraction) and the get path (recordIfSession), including +// the recording gate on the direct-iOS get fast path. + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ + mockRunAppleRunnerCommand: vi.fn(), +})); + +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + dispatchCommand: vi.fn(async () => ({})), + }; +}); + +vi.mock('../../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + runAppleRunnerCommand: mockRunAppleRunnerCommand, + }; +}); + +import { dispatchCommand } from '../../../core/dispatch.ts'; +const mockDispatch = vi.mocked(dispatchCommand); + +const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); + +beforeEach(() => { + mockDispatch.mockReset(); + mockDispatch.mockResolvedValue({}); + mockRunAppleRunnerCommand.mockReset(); + mockRunAppleRunnerCommand.mockResolvedValue({}); +}); + +const SAVE_BUTTON_NODES: RawSnapshotNode[] = [ + { + index: 0, + type: 'XCUIElementTypeButton', + identifier: 'save', + label: 'Save', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, +]; + +function makeSessionWithSnapshot( + sessionName: string, + options: { recordSession: boolean }, +): SessionState { + const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); + session.recordSession = options.recordSession; + session.snapshot = { + nodes: attachRefs(SAVE_BUTTON_NODES), + createdAt: Date.now(), + backend: 'xctest', + }; + return session; +} + +async function runCommand( + sessionStore: ReturnType, + sessionName: string, + command: string, + positionals: string[], + flags: CommandFlags = {}, +) { + return await handleInteractionCommands({ + req: { token: 't', session: sessionName, command, positionals, flags }, + sessionName, + sessionStore, + contextFromFlags, + }); +} + +test('press @ref while recording attaches target-v1 evidence to the recorded action, never to the public response', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'recording-press'; + sessionStore.set(sessionName, makeSessionWithSnapshot(sessionName, { recordSession: true })); + + // verify:true forces full runtime resolution, which captures node/tree. + const response = await runCommand(sessionStore, sessionName, 'press', ['@e1'], { verify: true }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data).not.toHaveProperty('node'); + expect(response.data).not.toHaveProperty('preActionNodes'); + expect(response.data).not.toHaveProperty('targetEvidence'); + } + + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.targetEvidence).toEqual({ + id: 'save', + role: 'button', + label: 'Save', + ancestry: [], + sibling: 0, + viewportOrder: 0, + rect: { x: 10, y: 20, width: 100, height: 40 }, + verification: 'verified', + }); + expect(recordedAction?.result).not.toHaveProperty('node'); + expect(recordedAction?.result).not.toHaveProperty('preActionNodes'); +}); + +test('press @ref without recording never computes target-v1 evidence', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'non-recording-press'; + sessionStore.set(sessionName, makeSessionWithSnapshot(sessionName, { recordSession: false })); + + const response = await runCommand(sessionStore, sessionName, 'press', ['@e1'], { verify: true }); + + expect(response?.ok).toBe(true); + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.targetEvidence).toBeUndefined(); + expect(recordedAction?.result).not.toHaveProperty('node'); + expect(recordedAction?.result).not.toHaveProperty('preActionNodes'); +}); + +test('get text @ref while recording attaches target-v1 evidence to the recorded action, never to session history payloads', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'recording-get-ref'; + sessionStore.set(sessionName, makeSessionWithSnapshot(sessionName, { recordSession: true })); + + const response = await runCommand(sessionStore, sessionName, 'get', ['text', '@e1']); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data).not.toHaveProperty('preActionNodes'); + expect(response.data).not.toHaveProperty('targetEvidence'); + } + + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.targetEvidence).toEqual({ + id: 'save', + role: 'button', + label: 'Save', + ancestry: [], + sibling: 0, + viewportOrder: 0, + rect: { x: 10, y: 20, width: 100, height: 40 }, + verification: 'verified', + }); + expect(recordedAction?.result).not.toHaveProperty('node'); + expect(recordedAction?.result).not.toHaveProperty('preActionNodes'); +}); + +test('get text @ref without recording never computes target-v1 evidence', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'non-recording-get-ref'; + sessionStore.set(sessionName, makeSessionWithSnapshot(sessionName, { recordSession: false })); + + const response = await runCommand(sessionStore, sessionName, 'get', ['text', '@e1']); + + expect(response?.ok).toBe(true); + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.targetEvidence).toBeUndefined(); +}); + +test('get text simple iOS id selector while recording skips the direct runner query and records evidence from the snapshot path', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'recording-get-direct-gate'; + const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); + session.recordSession = true; + sessionStore.set(sessionName, session); + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + return { + backend: 'xctest', + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 393, height: 852 }, + enabled: true, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'TextField', + label: 'Name', + identifier: 'field-name', + value: 'Ada Lovelace', + rect: { x: 24, y: 220, width: 320, height: 48 }, + enabled: true, + hittable: true, + }, + ], + }; + } + return {}; + }); + + const response = await runCommand(sessionStore, sessionName, 'get', ['text', 'id="field-name"']); + + expect(response?.ok).toBe(true); + // The direct-iOS querySelector fast path must be gated during recording so + // the snapshot path supplies the evidence tree. + expect(mockRunAppleRunnerCommand).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ command: 'querySelector' }), + expect.anything(), + ); + expect(mockDispatch.mock.calls.map((call) => call[1])).toContain('snapshot'); + + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.targetEvidence).toMatchObject({ + id: 'field-name', + role: 'textfield', + label: 'Name', + ancestry: [{ role: 'application' }], + verification: 'verified', + }); +}); diff --git a/src/daemon/handlers/interaction-common.ts b/src/daemon/handlers/interaction-common.ts index bacf1ae8c..218b9abf9 100644 --- a/src/daemon/handlers/interaction-common.ts +++ b/src/daemon/handlers/interaction-common.ts @@ -13,6 +13,7 @@ import { stripInternalInteractionFlags, } from '../interaction-outcome-policy.ts'; import { markPostGestureStabilization } from '../post-gesture-stabilization.ts'; +import { computeTargetEvidence, type RecordedTargetCapture } from '../session-target-evidence.ts'; export type ContextFromFlags = ( flags: CommandFlags | undefined, @@ -37,6 +38,8 @@ export function finalizeTouchInteraction(params: { flags: CommandFlags | undefined; result: Record; responseData: Record; + /** ADR 0012 decision 3: record-time input for the `target-v1` annotation. */ + recordedTarget?: RecordedTargetCapture; actionStartedAt: number; actionFinishedAt: number; androidFreshnessBaseline?: SnapshotState | undefined; @@ -50,16 +53,20 @@ export function finalizeTouchInteraction(params: { flags, result, responseData, + recordedTarget, actionStartedAt, actionFinishedAt, androidFreshnessBaseline, } = params; const actionFlags = stripInternalInteractionFlags(flags); + const targetEvidence = + session.recordSession && recordedTarget ? computeTargetEvidence(recordedTarget) : undefined; sessionStore.recordAction(session, { command, positionals, flags: actionFlags ?? {}, result, + ...(targetEvidence ? { targetEvidence } : {}), }); markPendingInteractionOutcome({ session, diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index 3bdb03227..7a50e51f7 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -5,6 +5,7 @@ import type { PressCommandResult, SettleObservation, } from '../../contracts/interaction.ts'; +import type { RecordedTargetCapture } from '../session-target-evidence.ts'; import { successText } from '../../utils/success-text.ts'; import { interactionResultExtra } from './interaction-touch-targets.ts'; @@ -43,6 +44,8 @@ export type InteractionResponsePayloads = { result: Record; /** The public payload returned to the client. */ responseData: Record; + /** Typed side channel — never part of either serialized payload. */ + recordedTarget?: RecordedTargetCapture; }; export function buildInteractionResponseData(params: { @@ -125,7 +128,15 @@ export function buildInteractionResponseData(params: { visualization.warning = warning; responseData.warning = warning; } - return { result: visualization, responseData }; + return { result: visualization, responseData, ...recordedTargetCapture(result) }; +} + +function recordedTargetCapture( + result: InteractionRuntimeResult, +): Pick { + const node = 'node' in result ? result.node : undefined; + const preActionNodes = 'preActionNodes' in result ? result.preActionNodes : undefined; + return node && preActionNodes ? { recordedTarget: { node, preActionNodes } } : {}; } // Attaches refsGeneration inside the settle payload when the response is diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index db2a804b2..00e0fbd4e 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -314,6 +314,7 @@ function readDirectIosSelectorTapTarget(params: { const { session, commandLabel, target, flags } = params; if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; + if (session.recordSession) return null; if (hasNonDefaultClickOptions(flags)) return null; if (commandSupportsVerifyEvidence(commandLabel) && flags?.verify === true) return null; if (commandSupportsSettleObservation(commandLabel) && flags?.settle === true) return null; @@ -634,6 +635,7 @@ function readDirectIosSelectorFillTarget(params: { }): DirectIosSelectorTarget | null { const { session, target, flags } = params; if (target.kind !== 'selector') return null; + if (session.recordSession) return null; if (commandSupportsVerifyEvidence('fill') && flags?.verify === true) return null; if (commandSupportsSettleObservation('fill') && flags?.settle === true) return null; return readDirectSelectorWithMaestroFallback(session, target.selector, flags); @@ -668,9 +670,7 @@ async function dispatchRuntimeInteraction< afterRun?(result: TResult): Promise; buildPayloads( result: TResult, - ): - | { result: Record; responseData: Record } - | Promise<{ result: Record; responseData: Record }>; + ): InteractionResponsePayloads | Promise; }, ): Promise { const session = params.sessionStore.get(params.sessionName); @@ -688,7 +688,7 @@ async function dispatchRuntimeInteraction< }, ); const actionFinishedAt = Date.now(); - const { result, responseData } = await options.buildPayloads(runtimeResult); + const { result, responseData, recordedTarget } = await options.buildPayloads(runtimeResult); if (readiness.status === 'recovered') { // Append, don't clobber — the builder may already carry a warning // (e.g. stale-refs, #1076). @@ -708,6 +708,7 @@ async function dispatchRuntimeInteraction< flags: params.req.flags, result, responseData, + recordedTarget, actionStartedAt, actionFinishedAt, androidFreshnessBaseline: options.androidFreshnessBaseline, diff --git a/src/daemon/selector-recording.ts b/src/daemon/selector-recording.ts index 1536c9236..367bffb95 100644 --- a/src/daemon/selector-recording.ts +++ b/src/daemon/selector-recording.ts @@ -1,5 +1,6 @@ import type { DaemonRequest } from './types.ts'; import { SessionStore } from './session-store.ts'; +import { computeTargetEvidence, type RecordedTargetCapture } from './session-target-evidence.ts'; export function buildFindRecordResult( result: Record, @@ -86,14 +87,19 @@ export function recordIfSession( sessionName: string, req: DaemonRequest, result: Record, + /** ADR 0012 decision 3: record-time input for the `target-v1` annotation. */ + recordedTarget?: RecordedTargetCapture, ): void { const session = sessionStore.get(sessionName); if (!session) return; + const targetEvidence = + session.recordSession && recordedTarget ? computeTargetEvidence(recordedTarget) : undefined; sessionStore.recordAction(session, { command: req.command, positionals: req.positionals ?? [], flags: req.flags ?? {}, result, + ...(targetEvidence ? { targetEvidence } : {}), }); } diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index a5e682bb7..7dff9880e 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -171,6 +171,10 @@ export async function dispatchGetViaRuntime( params.sessionName, req, buildGetRecordResult(result, sub), + { + node: result.node, + preActionNodes: result.preActionNodes, + }, ); const data = toDaemonGetData(result); return staleRefsWarning ? { ...data, warning: staleRefsWarning } : data; @@ -290,23 +294,34 @@ export async function dispatchWaitViaRuntime( return await withSessionlessRunnerCleanup(session, device, execute); } +function readDirectIosGetSelector( + session: SessionState | undefined, + property: 'text' | 'attrs', + selectorExpression: string, +): DirectIosSelectorTarget | null { + // ADR 0012 decision 3: recording requires the snapshot path so target + // evidence can be computed from the resolution tree. + if (!session || session.recordSession) return null; + const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); + // get text intentionally disambiguates label/text/value triplets from snapshots; the runner + // direct query rejects those ambiguous matches before the shared selector resolver can rank them. + if (property === 'text' && selector?.key !== 'id') return null; + return selector; +} + async function dispatchDirectIosSelectorGet( params: SelectorRuntimeParams, property: 'text' | 'attrs', selectorExpression: string, ): Promise { const session = params.sessionStore.get(params.sessionName); - const selector = readSimpleIosSelectorTarget({ session, selectorExpression }); + const selector = readDirectIosGetSelector(session, property, selectorExpression); if (!session || !selector) return null; - // get text intentionally disambiguates label/text/value triplets from snapshots; the runner - // direct query rejects those ambiguous matches before the shared selector resolver can rank them. - if (property === 'text' && selector.key !== 'id') return null; const result = await queryDirectIosSelectorOrFallback(params, session, selector); if (isDirectIosSelectorErrorResult(result)) return result.response; if (!result) return null; - const directQuery = { session, selector, result }; - const payload = buildDirectIosGetResult(property, directQuery.selector.raw, directQuery.result); + const payload = buildDirectIosGetResult(property, selector.raw, result); if (!payload) return null; recordIfSession( params.sessionStore, diff --git a/src/daemon/session-action-recorder.ts b/src/daemon/session-action-recorder.ts index 60fd67c18..d40493520 100644 --- a/src/daemon/session-action-recorder.ts +++ b/src/daemon/session-action-recorder.ts @@ -3,6 +3,7 @@ import { SCREENSHOT_ACTION_FLAG_KEYS } from '../contracts/screenshot.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; import type { SessionAction, SessionRuntimeHints, SessionState } from './types.ts'; import { expandSessionPath } from './session-paths.ts'; +import type { TargetAnnotationV1 } from '../replay/target-identity.ts'; export type RecordActionEntry = { command: string; @@ -10,6 +11,7 @@ export type RecordActionEntry = { flags: CommandFlags; runtime?: SessionRuntimeHints; result?: Record; + targetEvidence?: TargetAnnotationV1; }; export function recordActionEntry( @@ -30,6 +32,7 @@ export function recordActionEntry( runtime: entry.runtime, flags: sanitizeFlags(entry.flags), result: entry.result, + ...(entry.targetEvidence ? { targetEvidence: entry.targetEvidence } : {}), }; session.actions.push(action); emitDiagnostic({ diff --git a/src/daemon/session-script-writer.ts b/src/daemon/session-script-writer.ts index af1738b52..b66e85206 100644 --- a/src/daemon/session-script-writer.ts +++ b/src/daemon/session-script-writer.ts @@ -3,7 +3,10 @@ import path from 'node:path'; import { publicPlatformString } from '../kernel/device.ts'; import { inferFillText } from './action-utils.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; -import { formatPortableActionLine } from '../replay/script-formatting.ts'; +import { + formatPortableActionLine, + formatTargetAnnotationLines, +} from '../replay/script-formatting.ts'; import { expandSessionPath, safeSessionName } from './session-paths.ts'; import { appendScriptSeriesFlags, @@ -164,6 +167,7 @@ function formatScript(session: SessionState, actions: SessionAction[]): string { ); for (const action of actions) { if (action.flags?.noRecord) continue; + lines.push(...formatTargetAnnotationLines(action)); lines.push(formatActionLine(action)); } return `${lines.join('\n')}\n`; diff --git a/src/daemon/session-target-evidence.ts b/src/daemon/session-target-evidence.ts new file mode 100644 index 000000000..3ac772901 --- /dev/null +++ b/src/daemon/session-target-evidence.ts @@ -0,0 +1,309 @@ +/** + * ADR 0012 decision 3: record-time computation of `.ad` target-binding + * evidence (the `# agent-device:target-v1 {...}` annotation). + * + * `computeTargetEvidence` runs decision 3's "Record-time write" steps 1-5 + * against the tree the resolver already captured; it never captures, and + * callers gate it on `session.recordSession`. Tree-agnostic spec pieces live + * in `src/replay/target-identity.ts`, shared with the parser. + */ + +import type { SnapshotNode } from '../kernel/snapshot.ts'; +import { resolveRectCenter } from '../utils/rect-center.ts'; +import { normalizeType } from '../snapshot/snapshot-processing.ts'; +import { findNearestScrollableContainer } from './snapshot-presentation/tree.ts'; +import { + classifyTargetBindingMatch, + matchesAncestryPrefix, + matchesLocalIdentity, + normalizeIdentifierField, + normalizeLabelField, + normalizeRoleField, + serializeTargetAnnotationV1, + truncateToUtf8Bytes, + utf8ByteLength, + TARGET_ANNOTATION_MAX_ANCESTRY, + TARGET_ANNOTATION_MAX_FIELD_BYTES, + TARGET_ANNOTATION_MAX_PAYLOAD_BYTES, + type LocalIdentity, + type TargetAncestryEntry, + type TargetAnnotationV1, + type TargetScrollRegion, + type TargetVerification, +} from '../replay/target-identity.ts'; + +/** ADR 0012 decision 3: the resolved winner and the tree it was resolved from. */ +export type RecordedTargetCapture = { + node: SnapshotNode; + preActionNodes: SnapshotNode[]; +}; + +export function computeTargetEvidence( + capture: RecordedTargetCapture, +): TargetAnnotationV1 | undefined { + const { node, preActionNodes: nodes } = capture; + if (typeof node.index !== 'number') return undefined; + const byIndex = buildIndexMap(nodes); + const identity = boundedLocalIdentity(node); + const ancestryWalk = buildAncestryChain(node, byIndex, TARGET_ANNOTATION_MAX_ANCESTRY); + const fullAncestry = ancestryWalk.chain; + const sibling = computeSiblingOrdinal(nodes, node); + const scrollRegion = computeScrollRegionKey(node, byIndex); + const rect = boundedRect(node); + + // Decision 3's writer-parser invariant: reduce ancestry from the root side + // until the payload fits, stopping once only `ancestry[0]` is retained + // (floor 0 for a root node with no ancestors). + const floor = fullAncestry.length > 0 ? 1 : 0; + const buildCandidate = (ancestryLength: number) => { + const ancestry = fullAncestry.slice(0, ancestryLength); + const domain = computeDisambiguationDomain({ + nodes, + byIndex, + node, + identity, + ancestry, + sibling, + scrollRegion, + }); + const candidate: TargetAnnotationV1 = { + ...identity, + ancestry, + sibling, + viewportOrder: domain.viewportOrder, + ...(scrollRegion ? { scrollRegion } : {}), + ...(rect ? { rect } : {}), + verification: 'verified', + }; + return { candidate, domain }; + }; + + for (let ancestryLength = fullAncestry.length; ancestryLength >= floor; ancestryLength -= 1) { + const { candidate, domain } = buildCandidate(ancestryLength); + // Size against the longest verification value so the payload fits + // whichever one the self-check returns. + if ( + utf8ByteLength(serializeTargetAnnotationV1({ ...candidate, verification: 'unverifiable' })) <= + TARGET_ANNOTATION_MAX_PAYLOAD_BYTES + ) { + // A broken parent walk is a capture anomaly: fail closed instead of + // self-checking against structural signals that cannot be trusted. + candidate.verification = ancestryWalk.broken + ? 'unverifiable' + : runRecordTimeSelfCheck({ node, domain }); + return candidate; + } + if (ancestryLength === floor) { + // Decision 3's terminal fail-closed downgrade; rect is diagnostic-only + // and is dropped before ever emitting an over-cap payload. + candidate.verification = 'unverifiable'; + if ( + utf8ByteLength(serializeTargetAnnotationV1(candidate)) > TARGET_ANNOTATION_MAX_PAYLOAD_BYTES + ) { + delete candidate.rect; + } + return candidate; + } + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Identity / ancestry / sibling / scroll region — decision 3's structural +// primitives, computed once per candidate node against the record-time tree. +// --------------------------------------------------------------------------- + +function buildIndexMap(nodes: readonly SnapshotNode[]): Map { + const map = new Map(); + for (const node of nodes) map.set(node.index, node); + return map; +} + +/** The one identity reader: normalized AND field-capped, on every path. */ +function boundedLocalIdentity(node: SnapshotNode): LocalIdentity { + const role = normalizeRoleField(normalizeType(node.type ?? '')); + const id = normalizeIdentifierField(node.identifier); + const label = normalizeLabelField(node.label); + return { + ...(id !== undefined ? { id: truncateToUtf8Bytes(id, TARGET_ANNOTATION_MAX_FIELD_BYTES) } : {}), + role: truncateToUtf8Bytes(role, TARGET_ANNOTATION_MAX_FIELD_BYTES), + ...(label !== undefined + ? { label: truncateToUtf8Bytes(label, TARGET_ANNOTATION_MAX_FIELD_BYTES) } + : {}), + }; +} + +type AncestryWalk = { + chain: TargetAncestryEntry[]; + /** + * Decision 3 capture anomaly: a `parentIndex` that resolves to no node, or + * a parent cycle. A broken walk fails the annotation closed to + * `unverifiable` — the structural signals cannot be trusted. + */ + broken: boolean; +}; + +/** Decision 3 "Ancestry": nearest K ancestors, leaf→root, {role,label?}. */ +function buildAncestryChain( + node: SnapshotNode, + byIndex: Map, + limit: number, +): AncestryWalk { + const chain: TargetAncestryEntry[] = []; + const visited = new Set([node.index]); + let current = node; + while (chain.length < limit) { + if (typeof current.parentIndex !== 'number') return { chain, broken: false }; + const parent = byIndex.get(current.parentIndex); + if (!parent || visited.has(parent.index)) return { chain, broken: true }; + visited.add(parent.index); + const identity = boundedLocalIdentity(parent); + chain.push({ + role: identity.role, + ...(identity.label !== undefined ? { label: identity.label } : {}), + }); + current = parent; + } + return { chain, broken: false }; +} + +/** + * Decision 3 record-time write step 3: the winner's zero-based index among + * its OWN parent's children, in document order. Root-level nodes (no parent) + * are siblings of every other root-level node. + */ +function computeSiblingOrdinal(nodes: readonly SnapshotNode[], node: SnapshotNode): number { + const parentIndex = node.parentIndex; + let ordinal = 0; + for (const candidate of nodes) { + if (candidate.parentIndex !== parentIndex) continue; + if (candidate.index === node.index) return ordinal; + ordinal += 1; + } + return 0; +} + +/** Decision 3 record-time write step 4: nearest scrollable ancestor's local identity, or `undefined` for *none*. */ +function computeScrollRegionKey( + node: SnapshotNode, + byIndex: Map, +): TargetScrollRegion | undefined { + const container = findNearestScrollableContainer(node, byIndex); + if (!container) return undefined; + const identity = boundedLocalIdentity(container); + return { + role: identity.role, + ...(identity.id !== undefined + ? { id: identity.id } + : identity.label !== undefined + ? { label: identity.label } + : {}), + }; +} + +function scrollRegionKeysEqual( + a: TargetScrollRegion | undefined, + b: TargetScrollRegion | undefined, +): boolean { + if (!a && !b) return true; + if (!a || !b) return false; + return matchesLocalIdentity(a, b); +} + +function boundedRect(node: SnapshotNode): TargetAnnotationV1['rect'] { + const rect = node.rect; + if (!rect) return undefined; + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; +} + +// --------------------------------------------------------------------------- +// Disambiguation domain: decision 3 record-time write step 2 (identity set) +// + step 4 (scroll-region partition + viewportOrder), and everything the +// step-5 self-check needs to classify the result. +// --------------------------------------------------------------------------- + +type DisambiguationDomain = { + identitySet: SnapshotNode[]; + siblingMatches: SnapshotNode[]; + regionMembers: SnapshotNode[] | undefined; + orderedRegion: SnapshotNode[]; + viewportOrder: number; +}; + +function computeDisambiguationDomain(params: { + nodes: readonly SnapshotNode[]; + byIndex: Map; + node: SnapshotNode; + identity: LocalIdentity; + ancestry: TargetAncestryEntry[]; + sibling: number; + scrollRegion: TargetScrollRegion | undefined; +}): DisambiguationDomain { + const { nodes, byIndex, node, identity, ancestry, sibling, scrollRegion } = params; + + // Step 2: all nodes sharing the winner's local identity with a matching + // leaf-anchored ancestry prefix. + const identitySet = nodes.filter((candidate) => { + if (!matchesLocalIdentity(boundedLocalIdentity(candidate), identity)) return false; + const observed = buildAncestryChain(candidate, byIndex, Math.max(ancestry.length, 1)); + // A candidate with a broken parent walk cannot prove the prefix. + return !observed.broken && matchesAncestryPrefix(observed.chain, ancestry); + }); + + const siblingMatches = identitySet.filter( + (candidate) => computeSiblingOrdinal(nodes, candidate) === sibling, + ); + + const regionMembers = + identitySet.length > 0 + ? identitySet.filter((candidate) => + scrollRegionKeysEqual(computeScrollRegionKey(candidate, byIndex), scrollRegion), + ) + : undefined; + + const orderedRegion = regionMembers ? orderByViewportPosition(regionMembers) : []; + const viewportOrder = Math.max( + orderedRegion.findIndex((candidate) => candidate.index === node.index), + 0, + ); + + return { identitySet, siblingMatches, regionMembers, orderedRegion, viewportOrder }; +} + +/** Decision 3: rect center top-to-bottom then left-to-right; ties by document order; rect-less last, in document order. */ +function orderByViewportPosition(members: readonly SnapshotNode[]): SnapshotNode[] { + return members + .map((node, documentOrder) => ({ node, documentOrder, center: resolveRectCenter(node.rect) })) + .sort((a, b) => { + if (!a.center && !b.center) return a.documentOrder - b.documentOrder; + if (!a.center) return 1; + if (!b.center) return -1; + if (a.center.y !== b.center.y) return a.center.y - b.center.y; + if (a.center.x !== b.center.x) return a.center.x - b.center.x; + return a.documentOrder - b.documentOrder; + }) + .map((entry) => entry.node); +} + +/** + * Decision 3 record-time write step 5: run the shared replay-time + * classification (`classifyTargetBindingMatch`) against the record-time tree + * itself. + */ +function runRecordTimeSelfCheck(params: { + node: SnapshotNode; + domain: DisambiguationDomain; +}): TargetVerification { + const { node, domain } = params; + const winnerRef = node.ref; + const identitySetRefs = domain.identitySet.map((n) => n.ref); + const classification = classifyTargetBindingMatch({ + winnerRef, + matchedRefs: identitySetRefs, + identitySetRefs, + siblingMatchRefs: domain.siblingMatches.map((n) => n.ref), + regionMemberRefs: domain.regionMembers?.map((n) => n.ref), + viewportCandidateRef: domain.orderedRegion[domain.viewportOrder]?.ref, + }); + return classification.outcome; +} diff --git a/src/daemon/snapshot-presentation/tree.ts b/src/daemon/snapshot-presentation/tree.ts index e8b259056..8f4a87b69 100644 --- a/src/daemon/snapshot-presentation/tree.ts +++ b/src/daemon/snapshot-presentation/tree.ts @@ -71,11 +71,8 @@ function getDescendantEndPositions(nodes: RawSnapshotNode[]): number[] { return endPositions; } -function collectAncestors( - node: RawSnapshotNode, - byIndex: Map, -): RawSnapshotNode[] { - const ancestors: RawSnapshotNode[] = []; +function collectAncestors(node: T, byIndex: Map): T[] { + const ancestors: T[] = []; let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; const visited = new Set(); while (current && !visited.has(current.index)) { @@ -87,11 +84,11 @@ function collectAncestors( return ancestors; } -export function findNearestAncestor( - node: RawSnapshotNode, - byIndex: Map, - predicate: (ancestor: RawSnapshotNode) => boolean, -): RawSnapshotNode | null { +export function findNearestAncestor( + node: T, + byIndex: Map, + predicate: (ancestor: T) => boolean, +): T | null { for (const ancestor of collectAncestors(node, byIndex)) { if (predicate(ancestor)) { return ancestor; @@ -100,11 +97,11 @@ export function findNearestAncestor( return null; } -export function findNearestScrollableContainer( - node: RawSnapshotNode, - byIndex: Map, +export function findNearestScrollableContainer( + node: T, + byIndex: Map, options: { includeSelf?: boolean } = {}, -): RawSnapshotNode | null { +): T | null { const self = options.includeSelf === true && isScrollableSnapshotType(node.type) ? node : null; return ( self ?? diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 70a84efcd..9098084be 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -19,6 +19,7 @@ import type { RecordingScope } from '../core/recording-scope.ts'; import type { DeviceInfo, Platform, PlatformSelector } from '../kernel/device.ts'; import type { ExecBackgroundResult, ExecResult } from '../utils/exec.ts'; import type { SnapshotState } from '../kernel/snapshot.ts'; +import type { TargetAnnotationV1 } from '../replay/target-identity.ts'; import type { AppLogFailure, AppLogState } from './app-log-process.ts'; import type { DeviceLease } from './lease-registry.ts'; import type { AndroidNativePerfSession } from '../platforms/android/perf.ts'; @@ -398,4 +399,10 @@ export type SessionAction = { noRecord?: boolean; }; result?: Record; + /** + * ADR 0012 decision 3: parsed or record-time-computed `target-v1` + * evidence, written as a comment immediately before this action's line. + * Inert until migration step 4 adds enforcement. + */ + targetEvidence?: TargetAnnotationV1; }; diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index d4e1eb994..5c75a57fc 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -10,6 +10,7 @@ import { REPLAY_METADATA_PLATFORMS, writeReplayScript, } from '../script.ts'; +import type { TargetAnnotationV1 } from '../target-identity.ts'; import type { SessionAction, SessionState } from '../../daemon/types.ts'; function makeSession(): SessionState { @@ -373,3 +374,114 @@ test('replay parsing strips versioned-ref pins from recorded refs (#1076)', () = const malformed = parseReplayScriptDetailed('press @e2~x3').actions[0]; assert.deepEqual(malformed?.positionals, ['@e2~x3']); }); + +// --------------------------------------------------------------------------- +// ADR 0012 decision 3: `.ad` target-v1 annotation parsing/binding/preservation +// (migration step 3 — parser/writer only, no replay-time enforcement). +// --------------------------------------------------------------------------- + +const SAVE_EVIDENCE: TargetAnnotationV1 = { + id: 'save', + role: 'button', + label: 'Save', + ancestry: [{ role: 'toolbar', label: 'Editor' }, { role: 'window' }], + sibling: 0, + viewportOrder: 0, + scrollRegion: { role: 'scrollview', id: 'editor-scroll' }, + verification: 'verified', +}; + +const SAVE_EVIDENCE_LINE = + '# agent-device:target-v1 {"id":"save","role":"button","label":"Save","ancestry":[{"role":"toolbar","label":"Editor"},{"role":"window"}],"sibling":0,"viewportOrder":0,"scrollRegion":{"role":"scrollview","id":"editor-scroll"},"verification":"verified"}'; + +test('a target-v1 annotation immediately preceding an action line attaches to that action', () => { + const script = [ + 'context platform=ios device=iPhone', + SAVE_EVIDENCE_LINE, + 'click @e12 "Save"', + ].join('\n'); + const { actions } = parseReplayScriptDetailed(script); + assert.equal(actions.length, 1); + assert.deepEqual(actions[0]?.targetEvidence, SAVE_EVIDENCE); +}); + +test('a target-v1 annotation followed by a blank line before the action is rejected as INVALID_ARGS', () => { + const script = [SAVE_EVIDENCE_LINE, '', 'click @e12 "Save"'].join('\n'); + assert.throws( + () => parseReplayScriptDetailed(script), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /must be immediately followed by its action line/.test(error.message), + ); +}); + +test('a target-v1 annotation followed by another comment before the action is rejected as INVALID_ARGS', () => { + const script = [SAVE_EVIDENCE_LINE, '# note', 'click @e12 "Save"'].join('\n'); + assert.throws( + () => parseReplayScriptDetailed(script), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); +}); + +test('a target-v1 annotation as the last line of the script (no action follows) is rejected as INVALID_ARGS', () => { + assert.throws( + () => parseReplayScriptDetailed(SAVE_EVIDENCE_LINE), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); +}); + +test('a malformed target-v1 payload is rejected as INVALID_ARGS, not silently dropped', () => { + const script = ['# agent-device:target-v1 {not json', 'click @e12 "Save"'].join('\n'); + assert.throws( + () => parseReplayScriptDetailed(script), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); +}); + +test('an unknown future target-vN comment is an ordinary comment: no binding requirement, no evidence attached', () => { + const script = ['# agent-device:target-v2 {"whatever":true}', '', 'click @e12 "Save"'].join('\n'); + const { actions } = parseReplayScriptDetailed(script); + assert.equal(actions.length, 1); + assert.equal(actions[0]?.targetEvidence, undefined); +}); + +test('old readers ignoring the comment execute the action unchanged: an action without a preceding annotation carries no targetEvidence', () => { + const { actions } = parseReplayScriptDetailed('click @e12 "Save"'); + assert.equal(actions[0]?.targetEvidence, undefined); +}); + +test('writeReplayScript (read-then-rewrite / heal path) preserves a v1 annotation in canonical form', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-script-target-v1-')); + const replayPath = path.join(root, 'flow.ad'); + const actions: SessionAction[] = [ + { + ts: Date.now(), + command: 'click', + positionals: ['@e12'], + flags: {}, + targetEvidence: SAVE_EVIDENCE, + }, + ]; + + writeReplayScript(replayPath, actions, makeSession()); + const script = fs.readFileSync(replayPath, 'utf8'); + const lines = script.trim().split('\n'); + assert.equal(lines.at(-2), SAVE_EVIDENCE_LINE); + assert.equal(lines.at(-1), 'click @e12'); + + // Round trip: re-parsing the rewritten script recovers the same evidence. + const reparsed = parseReplayScriptDetailed(script); + assert.deepEqual(reparsed.actions[0]?.targetEvidence, SAVE_EVIDENCE); +}); + +test('session-recorded actions without target evidence never gain a fabricated annotation on rewrite', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-script-target-v1-none-')); + const replayPath = path.join(root, 'flow.ad'); + const actions: SessionAction[] = [ + { ts: Date.now(), command: 'click', positionals: ['@e12'], flags: {} }, + ]; + writeReplayScript(replayPath, actions, makeSession()); + const script = fs.readFileSync(replayPath, 'utf8'); + assert.equal(/agent-device:target-v1/.test(script), false); +}); diff --git a/src/replay/__tests__/target-identity-classification.test.ts b/src/replay/__tests__/target-identity-classification.test.ts new file mode 100644 index 000000000..ba689680a --- /dev/null +++ b/src/replay/__tests__/target-identity-classification.test.ts @@ -0,0 +1,110 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { classifyTargetBindingMatch } from '../target-identity.ts'; + +// Decision 3's replay-time verification paths 2-6 are shared with the +// writer's record-time self-check and stay isolated from parser coverage. + +test('classifyTargetBindingMatch path 2: matchCount 0 is unverifiable (selector-miss)', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: [], + identitySetRefs: [], + siblingMatchRefs: [], + regionMemberRefs: undefined, + viewportCandidateRef: undefined, + }); + assert.deepEqual(result, { path: 2, outcome: 'unverifiable', reason: 'selector-miss' }); +}); + +test('classifyTargetBindingMatch path 3: empty identity set is unverifiable', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1'], + identitySetRefs: [], + siblingMatchRefs: [], + regionMemberRefs: undefined, + viewportCandidateRef: undefined, + }); + assert.deepEqual(result, { path: 3, outcome: 'unverifiable', reason: 'identity-set-empty' }); +}); + +test('classifyTargetBindingMatch path 4: unique identity-set member equal to winner is verified', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1'], + identitySetRefs: ['e1'], + siblingMatchRefs: ['e1'], + regionMemberRefs: undefined, + viewportCandidateRef: undefined, + }); + assert.deepEqual(result, { path: 4, outcome: 'verified' }); +}); + +test('classifyTargetBindingMatch path 5: unique identity-set member that is not the winner is unverifiable', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1'], + identitySetRefs: ['e9'], + siblingMatchRefs: [], + regionMemberRefs: undefined, + viewportCandidateRef: undefined, + }); + assert.deepEqual(result, { path: 5, outcome: 'unverifiable', reason: 'unique-but-wrong' }); +}); + +test('classifyTargetBindingMatch path 6: duplicate identity resolved by a unique sibling match', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1', 'e2'], + identitySetRefs: ['e1', 'e2'], + siblingMatchRefs: ['e1'], + regionMemberRefs: undefined, + viewportCandidateRef: undefined, + }); + assert.deepEqual(result, { path: 6, outcome: 'verified' }); +}); + +test('classifyTargetBindingMatch path 6: viewport order resolves repeated sibling ordinals', () => { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1', 'e2'], + identitySetRefs: ['e1', 'e2'], + siblingMatchRefs: ['e1', 'e2'], + regionMemberRefs: ['e2', 'e1'], + viewportCandidateRef: 'e1', + }); + assert.deepEqual(result, { path: 6, outcome: 'verified' }); +}); + +test('classifyTargetBindingMatch path 6: unavailable or out-of-range viewport evidence falls through', () => { + for (const [regionMemberRefs, viewportCandidateRef] of [ + [undefined, undefined], + [['e1', 'e2'], undefined], + ] as const) { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1', 'e2'], + identitySetRefs: ['e1', 'e2'], + siblingMatchRefs: ['e1', 'e2'], + regionMemberRefs, + viewportCandidateRef, + }); + assert.deepEqual(result, { path: 6, outcome: 'unverifiable', reason: 'no-signal-isolation' }); + } +}); + +test('classifyTargetBindingMatch path 6: a signal isolating a different node than the winner is distinct from fall-through', () => { + for (const params of [ + { siblingMatchRefs: ['e1', 'e2'], regionMemberRefs: ['e1', 'e2'], viewportCandidateRef: 'e2' }, + { siblingMatchRefs: ['e2'], regionMemberRefs: ['e1', 'e2'], viewportCandidateRef: 'e1' }, + ]) { + const result = classifyTargetBindingMatch({ + winnerRef: 'e1', + matchedRefs: ['e1', 'e2'], + identitySetRefs: ['e1', 'e2'], + ...params, + }); + assert.deepEqual(result, { path: 6, outcome: 'unverifiable', reason: 'signal-isolated-wrong' }); + } +}); diff --git a/src/replay/__tests__/target-identity.test.ts b/src/replay/__tests__/target-identity.test.ts new file mode 100644 index 000000000..c0966da52 --- /dev/null +++ b/src/replay/__tests__/target-identity.test.ts @@ -0,0 +1,366 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '../../kernel/errors.ts'; +import { + formatTargetAnnotationCommentLine, + matchesAncestryPrefix, + matchesLocalIdentity, + normalizeLabelField, + parseTargetAnnotationCommentLine, + parseTargetAnnotationV1Payload, + serializeTargetAnnotationV1, + truncateToUtf8Bytes, + TARGET_ANNOTATION_MAX_ANCESTRY, + TARGET_ANNOTATION_MAX_FIELD_BYTES, + TARGET_ANNOTATION_MAX_PAYLOAD_BYTES, + type TargetAnnotationV1, +} from '../target-identity.ts'; + +function baseEvidence(overrides: Partial = {}): TargetAnnotationV1 { + return { + id: 'save', + role: 'button', + label: 'Save', + ancestry: [{ role: 'toolbar', label: 'Editor' }, { role: 'window' }], + sibling: 0, + viewportOrder: 0, + scrollRegion: { role: 'scrollview', id: 'editor-scroll' }, + verification: 'verified', + ...overrides, + }; +} + +function assertInvalidArgs(fn: () => unknown, messagePattern?: RegExp): void { + assert.throws( + fn, + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + (!messagePattern || messagePattern.test(error.message)), + ); +} + +// --------------------------------------------------------------------------- +// Canonical serialization / field order +// --------------------------------------------------------------------------- + +test('serializeTargetAnnotationV1 uses the exact canonical field order from decision 3', () => { + const json = serializeTargetAnnotationV1(baseEvidence()); + assert.equal( + json, + '{"id":"save","role":"button","label":"Save","ancestry":[{"role":"toolbar","label":"Editor"},{"role":"window"}],"sibling":0,"viewportOrder":0,"scrollRegion":{"role":"scrollview","id":"editor-scroll"},"verification":"verified"}', + ); +}); + +test('formatTargetAnnotationCommentLine emits the ASCII # agent-device:target-v1 prefix', () => { + const line = formatTargetAnnotationCommentLine(baseEvidence()); + assert.ok(line.startsWith('# agent-device:target-v1 {')); +}); + +// --------------------------------------------------------------------------- +// Parse-write-parse round trip / semantic equality +// --------------------------------------------------------------------------- + +test('parse(serialize(evidence)) round trips to a semantically equal object', () => { + const evidence = baseEvidence({ rect: { x: 1, y: 2, width: 3, height: 4 } }); + const parsedBack = parseTargetAnnotationV1Payload(serializeTargetAnnotationV1(evidence)); + assert.deepEqual(parsedBack, evidence); +}); + +test('parseTargetAnnotationCommentLine accepts known fields in any JSON key order', () => { + const line = + '# agent-device:target-v1 {"verification":"verified","sibling":0,"role":"button","viewportOrder":2,"ancestry":[],"id":"save"}'; + const result = parseTargetAnnotationCommentLine(line); + assert.equal(result.kind, 'v1'); + if (result.kind !== 'v1') throw new Error('unreachable'); + assert.deepEqual(result.evidence, { + id: 'save', + role: 'button', + ancestry: [], + sibling: 0, + viewportOrder: 2, + verification: 'verified', + }); +}); + +test('parseTargetAnnotationCommentLine ignores unknown fields', () => { + const line = + '# agent-device:target-v1 {"role":"button","verification":"verified","futureField":{"nested":true}}'; + const result = parseTargetAnnotationCommentLine(line); + assert.equal(result.kind, 'v1'); + if (result.kind !== 'v1') throw new Error('unreachable'); + assert.equal((result.evidence as Record).futureField, undefined); +}); + +test('an unknown future target-vN annotation is an ordinary comment to a v1 reader', () => { + const result = parseTargetAnnotationCommentLine('# agent-device:target-v2 {"anything":"goes"}'); + assert.deepEqual(result, { kind: 'future-version' }); +}); + +test('a line that merely mentions the tag in prose is an ordinary comment', () => { + assert.deepEqual(parseTargetAnnotationCommentLine('# see agent-device:target-v1 docs'), { + kind: 'none', + }); + assert.deepEqual(parseTargetAnnotationCommentLine('# just a comment'), { kind: 'none' }); +}); + +// --------------------------------------------------------------------------- +// Normalization: NFC, label trim/collapse, normalized-role source +// --------------------------------------------------------------------------- + +test('normalizeLabelField NFC-normalizes, trims, and collapses internal whitespace', () => { + // "é" as e + combining acute (NFD) must normalize to the precomposed (NFC) form. + const nfd = 'Café'; + assert.equal(normalizeLabelField(` ${nfd} au lait `), 'Café au lait'); +}); + +test('normalizeLabelField treats a whitespace-only label as absent', () => { + assert.equal(normalizeLabelField(' '), undefined); +}); + +test('embedded quotes and backslashes in labels round trip losslessly', () => { + const evidence = baseEvidence({ label: 'Say "hi" \\ backslash', id: undefined }); + const json = serializeTargetAnnotationV1(evidence); + assert.ok(json.includes('\\"hi\\"')); + const parsed = parseTargetAnnotationV1Payload(json); + assert.equal(parsed.label, 'Say "hi" \\ backslash'); +}); + +test('Unicode labels (including astral code points) round trip losslessly', () => { + const evidence = baseEvidence({ label: '\u{1F600} café résumé', id: undefined }); + const parsed = parseTargetAnnotationV1Payload(serializeTargetAnnotationV1(evidence)); + assert.equal(parsed.label, '\u{1F600} café résumé'); +}); + +// --------------------------------------------------------------------------- +// Old/new reader compatibility +// --------------------------------------------------------------------------- + +test('a v1 reader treats an annotation with only role + verification as valid, defaulting the rest', () => { + const result = parseTargetAnnotationCommentLine( + '# agent-device:target-v1 {"role":"button","verification":"verified"}', + ); + assert.equal(result.kind, 'v1'); + if (result.kind !== 'v1') throw new Error('unreachable'); + assert.deepEqual(result.evidence, { + role: 'button', + ancestry: [], + sibling: 0, + viewportOrder: 0, + verification: 'verified', + }); +}); + +// --------------------------------------------------------------------------- +// Leaf-anchored ancestry prefix matching: root-side truncation + inserted +// wrapper mismatch +// --------------------------------------------------------------------------- + +test('matchesAncestryPrefix accepts an observed chain that is a superset on the root side (truncation)', () => { + const recorded = [{ role: 'toolbar', label: 'Editor' }]; + const observedFullDepth = [ + { role: 'toolbar', label: 'Editor' }, + { role: 'window' }, + { role: 'application' }, + ]; + assert.equal(matchesAncestryPrefix(observedFullDepth, recorded), true); +}); + +test('matchesAncestryPrefix rejects an inserted wrapper ancestor (structure is part of identity)', () => { + const recorded = [{ role: 'toolbar', label: 'Editor' }, { role: 'window' }]; + // A new wrapper view inserted directly above the target shifts every entry + // one level down — the leaf-anchored prefix no longer matches. + const observedWithInsertedWrapper = [ + { role: 'view' }, + { role: 'toolbar', label: 'Editor' }, + { role: 'window' }, + ]; + assert.equal(matchesAncestryPrefix(observedWithInsertedWrapper, recorded), false); +}); + +test('matchesAncestryPrefix rejects a shorter observed chain', () => { + const recorded = [{ role: 'toolbar' }, { role: 'window' }]; + assert.equal(matchesAncestryPrefix([{ role: 'toolbar' }], recorded), false); +}); + +test('matchesAncestryPrefix leaves an unconstrained (absent) recorded label unconstrained', () => { + const recorded = [{ role: 'toolbar' }]; + assert.equal(matchesAncestryPrefix([{ role: 'toolbar', label: 'anything' }], recorded), true); +}); + +// --------------------------------------------------------------------------- +// Local identity +// --------------------------------------------------------------------------- + +test('matchesLocalIdentity: a recorded id never matches a node without that id', () => { + assert.equal(matchesLocalIdentity({ role: 'button' }, { id: 'save', role: 'button' }), false); +}); + +test('matchesLocalIdentity: with no recorded id, role+label must both match, absent-absent counts as equal', () => { + assert.equal(matchesLocalIdentity({ role: 'button' }, { role: 'button' }), true); + assert.equal(matchesLocalIdentity({ role: 'button', label: 'Save' }, { role: 'button' }), false); +}); + +// --------------------------------------------------------------------------- +// Bounds: 256-byte fields, 4 KiB payload, 8-entry ancestry — parser REJECTS, +// never truncates. +// --------------------------------------------------------------------------- + +test('parser rejects a string field exceeding the 256-byte cap', () => { + const oversizedLabel = 'x'.repeat(TARGET_ANNOTATION_MAX_FIELD_BYTES + 1); + assertInvalidArgs( + () => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', label: oversizedLabel, verification: 'verified' }), + ), + /256-byte field cap/, + ); +}); + +test('parser rejects a payload exceeding the 4 KiB cap', () => { + // Every individual field stays within the 256-byte field cap, but 8 + // maxed-out ancestry entries plus maxed top-level/scrollRegion fields blow + // the 4 KiB payload ceiling collectively. + const maxLabel = 'x'.repeat(TARGET_ANNOTATION_MAX_FIELD_BYTES); + const ancestry = Array.from({ length: TARGET_ANNOTATION_MAX_ANCESTRY }, () => ({ + role: maxLabel, + label: maxLabel, + })); + const json = JSON.stringify({ + id: maxLabel, + role: maxLabel, + label: maxLabel, + ancestry, + scrollRegion: { role: maxLabel, id: maxLabel, label: maxLabel }, + verification: 'verified', + }); + assert.ok(Buffer.byteLength(json, 'utf8') > TARGET_ANNOTATION_MAX_PAYLOAD_BYTES); + assertInvalidArgs(() => parseTargetAnnotationV1Payload(json), /4096-byte payload cap/); +}); + +test('parser rejects more than 8 ancestry entries', () => { + const ancestry = Array.from({ length: TARGET_ANNOTATION_MAX_ANCESTRY + 1 }, () => ({ + role: 'view', + })); + assertInvalidArgs( + () => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', ancestry, verification: 'verified' }), + ), + /8-entry cap/, + ); +}); + +test('truncateToUtf8Bytes never splits a surrogate pair', () => { + const emoji = '\u{1F600}'; // 4 UTF-8 bytes, a surrogate pair in UTF-16 + const truncated = truncateToUtf8Bytes(`ab${emoji}`, 3); + assert.equal(Buffer.byteLength(truncated, 'utf8') <= 3, true); + // The budget (3 bytes) fits "ab" but not the 4-byte emoji — the whole + // surrogate pair must be dropped together, never split. + assert.equal(truncated, 'ab'); + assert.equal(/[\ud800-\udbff]$/.test(truncated), false); +}); + +// --------------------------------------------------------------------------- +// Malformed / unbound annotations +// --------------------------------------------------------------------------- + +test('parser rejects non-JSON payloads', () => { + assertInvalidArgs(() => parseTargetAnnotationV1Payload('{not json'), /valid JSON/); +}); + +test('parser rejects a JSON array or scalar payload', () => { + assertInvalidArgs(() => parseTargetAnnotationV1Payload('[]')); + assertInvalidArgs(() => parseTargetAnnotationV1Payload('"button"')); +}); + +test('parser rejects a wrong-typed known field', () => { + assertInvalidArgs(() => + parseTargetAnnotationV1Payload(JSON.stringify({ role: 42, verification: 'verified' })), + ); +}); + +test('parser rejects an invalid verification value', () => { + assertInvalidArgs(() => + parseTargetAnnotationV1Payload(JSON.stringify({ role: 'button', verification: 'maybe' })), + ); +}); + +test('parser rejects a negative or non-integer sibling/viewportOrder', () => { + assertInvalidArgs(() => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', sibling: -1, verification: 'verified' }), + ), + ); + assertInvalidArgs(() => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', viewportOrder: 1.5, verification: 'verified' }), + ), + ); +}); + +// --------------------------------------------------------------------------- +// rect is diagnostic only: parsed, bounded, but never a comparison input at +// this parser layer (there is no comparator here yet — decision 3's +// enforcement lands in a later migration step — this just proves the parser +// accepts/round-trips it as inert data). +// --------------------------------------------------------------------------- + +test('rect parses and round trips but carries no comparison semantics here', () => { + const evidence = baseEvidence({ rect: { x: 10, y: 20, width: 30, height: 40 } }); + const parsed = parseTargetAnnotationV1Payload(serializeTargetAnnotationV1(evidence)); + assert.deepEqual(parsed.rect, { x: 10, y: 20, width: 30, height: 40 }); +}); + +test('parser rejects a malformed rect', () => { + assertInvalidArgs(() => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', rect: { x: 1, y: 2 }, verification: 'verified' }), + ), + ); +}); + +// --------------------------------------------------------------------------- +// Role presence: the writer emits `role` unconditionally (top level, every +// ancestry entry, scrollRegion) — possibly as the empty string for a +// typeless node, which stays accepted. A MISSING role key can only come from +// a hand-edited/adversarial annotation and must be rejected, or step-4 +// enforcement could match anonymous wrapper nodes through an implicit +// empty-role identity. +// --------------------------------------------------------------------------- + +test('parser rejects a missing top-level role', () => { + assertInvalidArgs( + () => parseTargetAnnotationV1Payload(JSON.stringify({ verification: 'verified' })), + /"role" is required/, + ); +}); + +test('parser rejects a missing role in an ancestry entry and in scrollRegion', () => { + assertInvalidArgs( + () => + parseTargetAnnotationV1Payload( + JSON.stringify({ + role: 'button', + ancestry: [{ label: 'Editor' }], + verification: 'verified', + }), + ), + /"ancestry\[0\]\.role" is required/, + ); + assertInvalidArgs( + () => + parseTargetAnnotationV1Payload( + JSON.stringify({ role: 'button', scrollRegion: { id: 'list' }, verification: 'verified' }), + ), + /"scrollRegion\.role" is required/, + ); +}); + +test('parser accepts an explicit empty-string role (writer-legal for typeless nodes)', () => { + const parsed = parseTargetAnnotationV1Payload( + JSON.stringify({ role: '', ancestry: [{ role: '' }], verification: 'verified' }), + ); + assert.equal(parsed.role, ''); + assert.deepEqual(parsed.ancestry, [{ role: '' }]); +}); diff --git a/src/replay/script-formatting.ts b/src/replay/script-formatting.ts index 19453fc7b..cdcb2575d 100644 --- a/src/replay/script-formatting.ts +++ b/src/replay/script-formatting.ts @@ -6,6 +6,7 @@ import { appendScreenshotActionScriptArgs, appendSnapshotActionScriptArgs, } from './script-utils.ts'; +import { formatTargetAnnotationCommentLine } from './target-identity.ts'; import type { SessionAction } from '../daemon/types.ts'; export function formatPortableActionLine( @@ -30,3 +31,12 @@ export function formatPortableActionLine( } return parts.join(' '); } + +/** + * ADR 0012 decision 3: the `# agent-device:target-v1 {...}` line that must + * immediately precede this action's line, or `[]` when the action carries no + * target evidence. Shared by both script writers for one canonical form. + */ +export function formatTargetAnnotationLines(action: SessionAction): string[] { + return action.targetEvidence ? [formatTargetAnnotationCommentLine(action.targetEvidence)] : []; +} diff --git a/src/replay/script.ts b/src/replay/script.ts index e9feb4e4d..3e597323c 100644 --- a/src/replay/script.ts +++ b/src/replay/script.ts @@ -5,7 +5,7 @@ import { readScreenshotScriptFlag } from '../contracts/screenshot.ts'; import type { DeviceTarget, PlatformSelector } from '../kernel/device.ts'; import { PLATFORM_SELECTORS, publicPlatformString } from '../kernel/device.ts'; import { parseReplayOpenFlags } from './open-script.ts'; -import { formatPortableActionLine } from './script-formatting.ts'; +import { formatPortableActionLine, formatTargetAnnotationLines } from './script-formatting.ts'; import type { SessionAction, SessionState } from '../daemon/types.ts'; import { formatScriptStringLiteral, @@ -14,6 +14,7 @@ import { parseReplayRuntimeFlags, stripRecordedRefGeneration, } from './script-utils.ts'; +import { parseTargetAnnotationCommentLine } from './target-identity.ts'; import { REPLAY_VAR_KEY_RE } from './vars.ts'; // Replay metadata `context platform=` lines support every accepted `--platform` @@ -41,15 +42,47 @@ export type ParsedReplayScript = { actionLines: number[]; }; +type PendingTargetAnnotation = { evidence: SessionAction['targetEvidence']; line: number }; + +// fallow-ignore-next-line complexity export function parseReplayScriptDetailed(script: string): ParsedReplayScript { const actions: SessionAction[] = []; const actionLines: number[] = []; const lines = script.split(/\r?\n/); let sawAction = false; + let pending: PendingTargetAnnotation | undefined; + + const rejectUnbound = ( + annotation: PendingTargetAnnotation, + index: number, + why: string, + ): never => { + throw new AppError( + 'INVALID_ARGS', + `target-v1 annotation on line ${annotation.line} must be immediately followed by its action line (line ${index + 1} ${why}).`, + ); + }; + for (const [index, rawLine] of lines.entries()) { const trimmed = rawLine.trim(); - if (trimmed.length === 0 || trimmed.startsWith('#')) continue; + if (trimmed.length === 0) { + if (pending) rejectUnbound(pending, index, 'is blank'); + continue; + } + if (trimmed.startsWith('#')) { + const annotation = parseTargetAnnotationCommentLine(trimmed); + if (annotation.kind === 'v1') { + if (pending) rejectUnbound(pending, index, 'is another target-v1 annotation'); + pending = { evidence: annotation.evidence, line: index + 1 }; + continue; + } + // An ordinary or future-target-vN comment still counts as an + // intervening line for a pending annotation. + if (pending) rejectUnbound(pending, index, 'is a comment'); + continue; + } if (isReplayEnvLine(trimmed)) { + if (pending) rejectUnbound(pending, index, 'is an env directive'); if (sawAction) { throw new AppError( 'INVALID_ARGS', @@ -59,11 +92,24 @@ export function parseReplayScriptDetailed(script: string): ParsedReplayScript { continue; } const parsed = parseReplayScriptLine(rawLine); - if (!parsed) continue; + if (!parsed) { + if (pending) rejectUnbound(pending, index, 'did not parse as an action'); + continue; + } + if (pending) { + parsed.targetEvidence = pending.evidence; + pending = undefined; + } actions.push(parsed); actionLines.push(index + 1); sawAction = true; } + if (pending) { + throw new AppError( + 'INVALID_ARGS', + `target-v1 annotation on line ${pending.line} must be immediately followed by its action line (end of script reached).`, + ); + } return { actions, actionLines }; } @@ -475,6 +521,8 @@ export function writeReplayScript( ); } for (const action of actions) { + // ADR 0012 decision 3: rewrites preserve v1 annotations in canonical form. + lines.push(...formatTargetAnnotationLines(action)); lines.push(formatReplayActionLine(action)); } const serialized = `${lines.join('\n')}\n`; diff --git a/src/replay/target-identity.ts b/src/replay/target-identity.ts new file mode 100644 index 000000000..1740d7bec --- /dev/null +++ b/src/replay/target-identity.ts @@ -0,0 +1,458 @@ +/** + * ADR 0012 decision 3: versioned `.ad` target-binding evidence — the + * tree-agnostic spine shared by the writer + * (`src/daemon/session-target-evidence.ts`) and the parser + * (`src/replay/script.ts`). Owns the wire type, canonical field order, + * normalization, size caps, payload parsing/validation, and the record/ + * replay-shared classification core. Inert in migration step 3: nothing + * enforces parsed evidence at replay time until step 4. + */ + +import { AppError } from '../kernel/errors.ts'; + +const TARGET_ANNOTATION_TAG = 'agent-device:target-v1'; +// Captures the rest of the line verbatim: a line claiming the tag with a +// garbage payload is a malformed v1 annotation, never an ordinary comment. +const TARGET_ANNOTATION_LINE_RE = /^#\s*agent-device:target-v(\d+)(?:\s+(.*))?$/; + +export const TARGET_ANNOTATION_MAX_FIELD_BYTES = 256; +export const TARGET_ANNOTATION_MAX_PAYLOAD_BYTES = 4096; +export const TARGET_ANNOTATION_MAX_ANCESTRY = 8; + +export type TargetAncestryEntry = { role: string; label?: string }; +export type TargetScrollRegion = { role: string; id?: string; label?: string }; +export type TargetRect = { x: number; y: number; width: number; height: number }; +export type TargetVerification = 'verified' | 'unverifiable'; + +export type TargetAnnotationV1 = { + id?: string; + role: string; + label?: string; + ancestry: TargetAncestryEntry[]; + sibling: number; + viewportOrder: number; + scrollRegion?: TargetScrollRegion; + rect?: TargetRect; + verification: TargetVerification; +}; + +// --------------------------------------------------------------------------- +// Normalization (decision 3 "Normalization"): all strings NFC; `label` fields +// additionally trim and collapse internal whitespace runs. A string that is +// empty after normalization is omitted (writer) / treated as absent +// (comparator). Applies to every string field: top-level id/role/label, +// ancestry entry role/label, and scrollRegion role/id/label. +// --------------------------------------------------------------------------- + +function nfc(value: string): string { + return value.normalize('NFC'); +} + +/** id/role fields: NFC only (never trimmed/collapsed — see decision 3). */ +export function normalizeIdentifierField(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const normalized = nfc(value); + return normalized.length > 0 ? normalized : undefined; +} + +/** `role` is always required (may be the empty string per decision 3's ancestry note). */ +export function normalizeRoleField(value: string): string { + return nfc(value); +} + +/** label fields: NFC, trim, collapse internal whitespace runs to one space. */ +export function normalizeLabelField(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const collapsed = nfc(value).trim().replace(/\s+/g, ' '); + return collapsed.length > 0 ? collapsed : undefined; +} + +export function utf8ByteLength(value: string): number { + return Buffer.byteLength(value, 'utf8'); +} + +/** + * Writer-side field truncation to the 256-byte cap ("per-field truncation", + * decision 3's writer-parser invariant). Trims on a code-point boundary so a + * surrogate pair is never split. The parser never calls this — it REJECTS + * oversized fields instead (see `parseTargetAnnotationV1Payload`). + */ +export function truncateToUtf8Bytes(value: string, maxBytes: number): string { + if (utf8ByteLength(value) <= maxBytes) return value; + let end = value.length; + while (end > 0 && utf8ByteLength(value.slice(0, end)) > maxBytes) { + end -= 1; + } + if (end > 0) { + const code = value.charCodeAt(end - 1); + if (code >= 0xd8_00 && code <= 0xdb_ff) end -= 1; // don't split a surrogate pair + } + return value.slice(0, end); +} + +// --------------------------------------------------------------------------- +// Canonical serialization (decision 3's exact field order + nested-object +// key order from the example payload). +// --------------------------------------------------------------------------- + +function buildCanonicalTargetAnnotationObject( + evidence: TargetAnnotationV1, +): Record { + const obj: Record = {}; + if (evidence.id !== undefined) obj.id = evidence.id; + obj.role = evidence.role; + if (evidence.label !== undefined) obj.label = evidence.label; + obj.ancestry = evidence.ancestry.map(buildAncestryEntryObject); + obj.sibling = evidence.sibling; + obj.viewportOrder = evidence.viewportOrder; + if (evidence.scrollRegion) obj.scrollRegion = buildScrollRegionObject(evidence.scrollRegion); + if (evidence.rect) obj.rect = buildRectObject(evidence.rect); + obj.verification = evidence.verification; + return obj; +} + +function buildAncestryEntryObject(entry: TargetAncestryEntry): Record { + const obj: Record = { role: entry.role }; + if (entry.label !== undefined) obj.label = entry.label; + return obj; +} + +function buildScrollRegionObject(region: TargetScrollRegion): Record { + const obj: Record = { role: region.role }; + if (region.id !== undefined) obj.id = region.id; + if (region.label !== undefined) obj.label = region.label; + return obj; +} + +function buildRectObject(rect: TargetRect): Record { + return { x: rect.x, y: rect.y, width: rect.width, height: rect.height }; +} + +export function serializeTargetAnnotationV1(evidence: TargetAnnotationV1): string { + return JSON.stringify(buildCanonicalTargetAnnotationObject(evidence)); +} + +export function formatTargetAnnotationCommentLine(evidence: TargetAnnotationV1): string { + return `# ${TARGET_ANNOTATION_TAG} ${serializeTargetAnnotationV1(evidence)}`; +} + +// --------------------------------------------------------------------------- +// Parsing (decision 3's parser bullet + "Replay-time verification" intro). +// --------------------------------------------------------------------------- + +export type TargetAnnotationLineParseResult = + | { kind: 'none' } + | { kind: 'future-version' } + | { kind: 'v1'; evidence: TargetAnnotationV1 }; + +/** + * Recognizes a `# agent-device:target-vN {...}` comment line. `N !== 1` is an + * ordinary comment to this (v1) reader, per decision 3: "An unknown future + * `target-vN` comment is an ordinary comment to a v1 reader." Any other `#` + * line (including one that merely mentions the tag inside prose) is `none`. + */ +export function parseTargetAnnotationCommentLine(rawLine: string): TargetAnnotationLineParseResult { + const trimmed = rawLine.trim(); + if (!trimmed.startsWith('#')) return { kind: 'none' }; + const match = TARGET_ANNOTATION_LINE_RE.exec(trimmed); + if (!match) return { kind: 'none' }; + const version = Number(match[1]); + if (version !== 1) return { kind: 'future-version' }; + const evidence = parseTargetAnnotationV1Payload((match[2] ?? '').trim()); + return { kind: 'v1', evidence }; +} + +/** + * Parses and validates the JSON payload of a `target-v1` annotation. + * Accepts known fields in any order, ignores unknown fields, NFC-normalizes + * known strings, and rejects malformed/oversized payloads with + * `INVALID_ARGS` (decision 3: "The parser rejects a v1 annotation exceeding + * these bounds with INVALID_ARGS"). + */ +// fallow-ignore-next-line complexity +export function parseTargetAnnotationV1Payload(jsonText: string): TargetAnnotationV1 { + if (utf8ByteLength(jsonText) > TARGET_ANNOTATION_MAX_PAYLOAD_BYTES) { + throw new AppError( + 'INVALID_ARGS', + `target-v1 annotation exceeds the ${TARGET_ANNOTATION_MAX_PAYLOAD_BYTES}-byte payload cap.`, + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(jsonText); + } catch { + throw new AppError('INVALID_ARGS', 'target-v1 annotation is not valid JSON.'); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new AppError('INVALID_ARGS', 'target-v1 annotation must be a JSON object.'); + } + const raw = parsed as Record; + + const role = parseRequiredRoleField(raw.role, 'role'); + const id = parseOptionalIdentifierField(raw.id, 'id'); + const label = parseOptionalLabelField(raw.label, 'label'); + const ancestry = parseAncestryField(raw.ancestry); + const sibling = parseNonNegativeIntField(raw.sibling, 'sibling', 0); + const viewportOrder = parseNonNegativeIntField(raw.viewportOrder, 'viewportOrder', 0); + const scrollRegion = parseScrollRegionField(raw.scrollRegion); + const rect = parseRectField(raw.rect); + const verification = parseVerificationField(raw.verification); + + return { + ...(id !== undefined ? { id } : {}), + role, + ...(label !== undefined ? { label } : {}), + ancestry, + sibling, + viewportOrder, + ...(scrollRegion ? { scrollRegion } : {}), + ...(rect ? { rect } : {}), + verification, + }; +} + +/** + * The writer emits `role` unconditionally (possibly as the empty string for + * a typeless node), so a missing role key is always foreign input and is + * rejected rather than defaulted. + */ +function parseRequiredRoleField(value: unknown, field: string): string { + if (value === undefined) { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" is required.`); + } + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" must be a string.`); + } + return boundField(normalizeRoleField(value), field); +} + +function parseOptionalIdentifierField(value: unknown, field: 'id'): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" must be a string.`); + } + const normalized = normalizeIdentifierField(value); + return normalized === undefined ? undefined : boundField(normalized, field); +} + +function parseOptionalLabelField(value: unknown, field: string): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" must be a string.`); + } + const normalized = normalizeLabelField(value); + return normalized === undefined ? undefined : boundField(normalized, field); +} + +function boundField(value: string, field: string): string { + if (utf8ByteLength(value) > TARGET_ANNOTATION_MAX_FIELD_BYTES) { + throw new AppError( + 'INVALID_ARGS', + `target-v1 "${field}" exceeds the ${TARGET_ANNOTATION_MAX_FIELD_BYTES}-byte field cap.`, + ); + } + return value; +} + +function parseAncestryField(value: unknown): TargetAncestryEntry[] { + if (value === undefined) return []; + if (!Array.isArray(value)) { + throw new AppError('INVALID_ARGS', 'target-v1 "ancestry" must be an array.'); + } + if (value.length > TARGET_ANNOTATION_MAX_ANCESTRY) { + throw new AppError( + 'INVALID_ARGS', + `target-v1 "ancestry" exceeds the ${TARGET_ANNOTATION_MAX_ANCESTRY}-entry cap.`, + ); + } + return value.map((entry, index) => parseAncestryEntry(entry, index)); +} + +function parseAncestryEntry(entry: unknown, index: number): TargetAncestryEntry { + if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) { + throw new AppError('INVALID_ARGS', `target-v1 "ancestry[${index}]" must be an object.`); + } + const record = entry as Record; + const role = parseRequiredRoleField(record.role, `ancestry[${index}].role`); + const label = parseOptionalLabelField(record.label, `ancestry[${index}].label`); + return { role, ...(label !== undefined ? { label } : {}) }; +} + +function parseNonNegativeIntField(value: unknown, field: string, fallback: number): number { + if (value === undefined) return fallback; + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" must be a non-negative integer.`); + } + return value; +} + +function parseScrollRegionField(value: unknown): TargetScrollRegion | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new AppError('INVALID_ARGS', 'target-v1 "scrollRegion" must be an object.'); + } + const record = value as Record; + const role = parseRequiredRoleField(record.role, 'scrollRegion.role'); + const id = parseOptionalIdentifierField(record.id, 'id'); + const label = parseOptionalLabelField(record.label, 'scrollRegion.label'); + return { + role, + ...(id !== undefined ? { id } : {}), + ...(label !== undefined ? { label } : {}), + }; +} + +function parseRectField(value: unknown): TargetRect | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new AppError('INVALID_ARGS', 'target-v1 "rect" must be an object.'); + } + const record = value as Record; + const x = parseFiniteNumberField(record.x, 'rect.x'); + const y = parseFiniteNumberField(record.y, 'rect.y'); + const width = parseFiniteNumberField(record.width, 'rect.width'); + const height = parseFiniteNumberField(record.height, 'rect.height'); + return { x, y, width, height }; +} + +function parseFiniteNumberField(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new AppError('INVALID_ARGS', `target-v1 "${field}" must be a finite number.`); + } + return value; +} + +function parseVerificationField(value: unknown): TargetVerification { + if (value === 'verified' || value === 'unverifiable') return value; + throw new AppError( + 'INVALID_ARGS', + 'target-v1 "verification" must be "verified" or "unverifiable".', + ); +} + +// --------------------------------------------------------------------------- +// Local identity + ancestry-prefix matching (decision 3 "Local identity" / +// "Ancestry"). Pure over the small structural shapes above — no tree +// dependency, so both the writer (over `SnapshotNode`-derived values) and a +// future replay verifier can share it verbatim. +// --------------------------------------------------------------------------- + +export type LocalIdentity = { id?: string; role: string; label?: string }; + +/** + * Decision 3 "Local identity": id match wins outright when the recording + * carries one ("a recorded id never matches a node without that id"); with + * no recorded id, role+label must both match (label absent on both sides + * counts as equal; present on exactly one side is a mismatch). + */ +export function matchesLocalIdentity(candidate: LocalIdentity, recorded: LocalIdentity): boolean { + if (recorded.id !== undefined) return candidate.id === recorded.id; + return candidate.role === recorded.role && candidate.label === recorded.label; +} + +/** + * Decision 3 "Ancestry": leaf-anchored prefix match. `observed` must be at + * least as long as `recorded`; each recorded entry's role must match exactly + * and, when the recorded entry carries a label, so must the observed one (an + * absent recorded label is unconstrained). + */ +export function matchesAncestryPrefix( + observed: readonly TargetAncestryEntry[], + recorded: readonly TargetAncestryEntry[], +): boolean { + if (observed.length < recorded.length) return false; + for (const [index, entry] of recorded.entries()) { + const candidate = observed[index]; + if (!candidate) return false; + if (candidate.role !== entry.role) return false; + if (entry.label !== undefined && candidate.label !== entry.label) return false; + } + return true; +} + +// --------------------------------------------------------------------------- +// Classification core (decision 3 "Replay-time verification", paths 2-6; +// path 1 is the caller's pre-resolution check). Generic over node refs so +// the record-time self-check and future replay-time enforcement share it. +// --------------------------------------------------------------------------- + +export type TargetBindingClassificationInput = { + /** The node the resolver actually picked. */ + winnerRef: string; + /** `matchCount`'s domain: nodes matching the recorded selector/ref. */ + matchedRefs: readonly string[]; + /** Members of `matchedRefs` sharing the recorded local identity + ancestry prefix (decision 3 set I). */ + identitySetRefs: readonly string[]; + /** Members of `identitySetRefs` whose same-parent sibling ordinal equals the recorded `sibling`. */ + siblingMatchRefs: readonly string[]; + /** + * Members of `identitySetRefs` in the partition whose scroll-region key + * equals the recorded `scrollRegion` (the *none* partition when none was + * recorded). `undefined` when that region no longer exists at all. + */ + regionMemberRefs: readonly string[] | undefined; + /** `regionMemberRefs` ordered by decision 3's viewport ordering; the ref at the recorded `viewportOrder`, if in range. */ + viewportCandidateRef: string | undefined; +}; + +/** + * Decision 3 keeps two spec-distinct failure classes inside path 6, and the + * `reason` field preserves the distinction for migration step 4's divergence + * `kind` mapping: + * + * - a disambiguation signal ISOLATING exactly one member that differs from + * the winner is "compare with W as in paths 4/5" — the same class as path + * 5's unique-but-wrong rebind, i.e. a future `identity-mismatch` + * (`signal-isolated-wrong`); + * - neither signal isolating any member is the true fall-through — a future + * `identity-unverifiable` with up to 5 candidates (`no-signal-isolation`). + */ +export type TargetBindingClassification = + | { path: 2; outcome: 'unverifiable'; reason: 'selector-miss' } + | { path: 3; outcome: 'unverifiable'; reason: 'identity-set-empty' } + | { path: 4; outcome: 'verified' } + | { path: 5; outcome: 'unverifiable'; reason: 'unique-but-wrong' } + | { path: 6; outcome: 'verified' } + | { path: 6; outcome: 'unverifiable'; reason: 'signal-isolated-wrong' | 'no-signal-isolation' }; + +/** + * Decision 3 "Replay-time verification", paths 2-6. `matchCount == 0` (path + * 2), an empty identity set (path 3), a unique identity-set member that is or + * isn't the winner (paths 4/5), and the sibling → region-scoped-viewportOrder + * disambiguation cascade (path 6) — falling through to unverifiable, never a + * silent pick, exactly as decision 3 specifies. + */ +export function classifyTargetBindingMatch( + input: TargetBindingClassificationInput, +): TargetBindingClassification { + if (input.matchedRefs.length === 0) { + return { path: 2, outcome: 'unverifiable', reason: 'selector-miss' }; + } + if (input.identitySetRefs.length === 0) { + return { path: 3, outcome: 'unverifiable', reason: 'identity-set-empty' }; + } + if (input.identitySetRefs.length === 1) { + return input.identitySetRefs[0] === input.winnerRef + ? { path: 4, outcome: 'verified' } + : { path: 5, outcome: 'unverifiable', reason: 'unique-but-wrong' }; + } + if (input.siblingMatchRefs.length === 1) { + // The sibling signal isolates exactly one member: the evidence denotes + // it — compare with the winner as in paths 4/5 (decision 3, path 6.i). + return input.siblingMatchRefs[0] === input.winnerRef + ? { path: 6, outcome: 'verified' } + : { path: 6, outcome: 'unverifiable', reason: 'signal-isolated-wrong' }; + } + if ( + input.regionMemberRefs !== undefined && + input.regionMemberRefs.length > 0 && + input.viewportCandidateRef !== undefined + ) { + // Region-scoped viewportOrder denotes a member: compare as in paths 4/5 + // (decision 3, path 6.ii). + return input.viewportCandidateRef === input.winnerRef + ? { path: 6, outcome: 'verified' } + : { path: 6, outcome: 'unverifiable', reason: 'signal-isolated-wrong' }; + } + return { path: 6, outcome: 'unverifiable', reason: 'no-signal-isolation' }; +} diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index 3c609a4d7..a1b686d07 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -30,6 +30,7 @@ const CONTRACT_DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; export async function withIosContractDaemon( entries: readonly ProviderScenarioProviderEntry[], run: (daemon: ProviderScenarioHarness, transcript: ProviderScenarioTranscript) => Promise, + options: { saveScript?: boolean | string } = {}, ): Promise { const transcript = createProviderTranscript(entries); const appleRunnerProvider = createAppleRunnerProviderFromTranscript(transcript, 'ios.runner'); @@ -50,6 +51,7 @@ export async function withIosContractDaemon( const open = await daemon.callCommand('open', [CONTRACT_APP], { platform: 'ios', udid: CONTRACT_DEVICE_ID, + ...(options.saveScript !== undefined ? { saveScript: options.saveScript } : {}), }); assertRpcOk(open); await run(daemon, transcript); diff --git a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts index 21a26cce4..09cfb4311 100644 --- a/test/integration/interaction-contract/direct-ios-selector.contract.test.ts +++ b/test/integration/interaction-contract/direct-ios-selector.contract.test.ts @@ -16,6 +16,7 @@ import { runnerSnapshotEntry, runnerTapEntry, runnerTapErrorEntry, + runnerTypeEntry, withIosContractDaemon, } from './daemon-harness.ts'; @@ -28,6 +29,30 @@ import { const scenario = (guarantee: InteractionGuarantee): string => scenarioName(DIRECT_IOS_SELECTOR_COVERAGE, guarantee); +const RECORDING_TARGET_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + identifier: 'continue', + label: 'Continue', + rect: { x: 40, y: 160, width: 120, height: 44 }, + enabled: true, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'TextField', + identifier: 'email', + label: 'Email', + rect: { x: 40, y: 240, width: 280, height: 44 }, + enabled: true, + hittable: true, + }, +] as const; + test(scenario('responseConstruction'), async () => { await withIosContractDaemon([runnerTapEntry({ x: 150, y: 200 })], async (daemon, transcript) => { const click = await daemon.callCommand('click', ['label=Continue']); @@ -48,6 +73,52 @@ test(scenario('responseConstruction'), async () => { }); }); +test('recorded simple iOS selector click and fill use runtime resolution and persist target-v1 evidence', async () => { + await withIosContractDaemon( + [ + runnerSnapshotEntry(RECORDING_TARGET_NODES), + runnerTapEntry({ x: 100, y: 182 }), + runnerSnapshotEntry(RECORDING_TARGET_NODES), + runnerTypeEntry({ x: 180, y: 262 }), + ], + async (daemon, transcript) => { + assertRpcOk(await daemon.callCommand('click', ['id=continue'])); + assertRpcOk(await daemon.callCommand('fill', ['id=email', 'ada@example.com'])); + + assert.equal(transcript.calls[0]?.command, 'ios.runner.snapshot'); + assert.equal(transcript.calls[1]?.command, 'ios.runner.tap'); + assert.equal(transcript.calls[2]?.command, 'ios.runner.snapshot'); + assert.equal(transcript.calls[3]?.command, 'ios.runner.type'); + for (const call of [transcript.calls[1], transcript.calls[3]]) { + const request = call?.request as Record | undefined; + assert.equal(request?.selectorKey, undefined); + } + + const actions = daemon + .session() + ?.actions.filter((action) => action.command === 'click' || action.command === 'fill'); + assert.deepEqual( + actions?.map((action) => action.command), + ['click', 'fill'], + ); + const targetIdentities = actions?.map((action) => { + const evidence = action.targetEvidence as Record | undefined; + return { + id: evidence?.id, + role: evidence?.role, + label: evidence?.label, + verification: evidence?.verification, + }; + }); + assert.deepEqual(targetIdentities, [ + { id: 'continue', role: 'button', label: 'Continue', verification: 'verified' }, + { id: 'email', role: 'textfield', label: 'Email', verification: 'verified' }, + ]); + }, + { saveScript: true }, + ); +}); + test(scenario('offscreen'), async () => { await withIosContractDaemon( [ diff --git a/test/integration/interaction-contract/native-ref.contract.test.ts b/test/integration/interaction-contract/native-ref.contract.test.ts index a167b1637..4a3d70d18 100644 --- a/test/integration/interaction-contract/native-ref.contract.test.ts +++ b/test/integration/interaction-contract/native-ref.contract.test.ts @@ -4,6 +4,7 @@ import type { InteractionGuarantee } from '../../../src/contracts/interaction-gu import type { SnapshotState } from '../../../src/kernel/snapshot.ts'; import { ref } from '../../../src/commands/index.ts'; import { scenarioName } from './coverage-manifest.ts'; +import { buildInteractionResponseData } from '../../../src/daemon/handlers/interaction-touch-response.ts'; import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; import { closedDrawerSnapshot, @@ -169,5 +170,25 @@ test(scenario('responseConstruction'), async () => { assert.deepEqual(result.target, { kind: 'ref', ref: '@e1' }); assert.deepEqual(result.backendResult, { ref: 'e1' }); assert.equal(result.point, undefined); - assert.equal(result.node, undefined); + // ADR 0012 decision 3: the preflight's guard lookup supplies the + // record-time evidence node on the runtime result. + assert.equal(result.node?.ref, 'e1'); + assert.ok(Array.isArray(result.preActionNodes)); + + const { + result: visualization, + responseData, + recordedTarget, + } = buildInteractionResponseData({ + source: { kind: 'runtime', result }, + referenceFrame: undefined, + }); + // The construction site routes node/tree onto the typed recordedTarget + // channel only; neither serialized payload carries them. + assert.equal(recordedTarget?.node.ref, 'e1'); + for (const payload of [visualization, responseData]) { + assert.equal('node' in payload, false); + assert.equal('preActionNodes' in payload, false); + assert.equal('targetEvidence' in payload, false); + } });