diff --git a/scripts/maestro-conformance/differential/flows/optional-scroll-and-wait.yaml b/scripts/maestro-conformance/differential/flows/optional-scroll-and-wait.yaml new file mode 100644 index 0000000000..e1c36c7935 --- /dev/null +++ b/scripts/maestro-conformance/differential/flows/optional-scroll-and-wait.yaml @@ -0,0 +1,19 @@ +# Layer-3 device flow. Optional scrollUntilVisible and extendedWaitUntil commands +# that target a missing element must warn and continue to the next step on both +# engines. A failed-instead-of-warned classification fails the flow. +appId: com.callstack.agentdevicelab +--- +- launchApp: + clearState: true +- assertVisible: Agent Device Tester +- scrollUntilVisible: + element: + id: this-element-never-exists + optional: true + timeout: 1000 +- extendedWaitUntil: + visible: + id: this-element-never-exists + optional: true + timeout: 1000 +- assertVisible: Agent Device Tester diff --git a/scripts/maestro-conformance/differential/scenarios.ts b/scripts/maestro-conformance/differential/scenarios.ts index c89fa0a6d3..f90e7f47b7 100644 --- a/scripts/maestro-conformance/differential/scenarios.ts +++ b/scripts/maestro-conformance/differential/scenarios.ts @@ -189,4 +189,13 @@ export const DIFFERENTIAL_SCENARIOS: DifferentialScenario[] = [ divergenceMeans: 'agent-device fails the tap/wait/tap sequence with a stability-generation mismatch where upstream passes.', }, + { + id: 'optional-warned-scroll-and-wait', + flow: 'differential/flows/optional-scroll-and-wait.yaml', + comparesAcrossEngines: + 'Optional scrollUntilVisible and extendedWaitUntil commands that fail to find their targets are downgraded to warnings and the flow continues to the next step on both engines — a failed-instead-of-warned classification flips the exit code, so outcome parity proves this.', + expect: 'pass', + divergenceMeans: + 'agent-device failed an optional scrollUntilVisible or extendedWaitUntil command instead of warning and continuing.', + }, ]; diff --git a/scripts/maestro-conformance/expected-divergence.ts b/scripts/maestro-conformance/expected-divergence.ts index a75f3e3b54..ed25ea5c2a 100644 --- a/scripts/maestro-conformance/expected-divergence.ts +++ b/scripts/maestro-conformance/expected-divergence.ts @@ -91,9 +91,9 @@ export const FLOW_DIVERGENCES: Record = { }, 'upstream/076_optional_assertion': { classification: 'we-reject', - reason: 'optional is supported on tapOn/assertion targets; the flow marks scrollUntilVisible/extendedWaitUntil optional and uses assertTrue.', - unsupported: ['optional (scrollUntilVisible/extendedWaitUntil)', 'assertTrue'], - tracking: COMPAT_TRACKER, + reason: 'assertTrue is outside the supported subset; optional is now supported on scrollUntilVisible and extendedWaitUntil.', + unsupported: ['assertTrue'], + tracking: 'https://github.com/callstack/agent-device/issues/1295', }, 'upstream/079_scroll_until_visible': { classification: 'we-reject', diff --git a/src/compat/maestro/__tests__/engine.test.ts b/src/compat/maestro/__tests__/engine.test.ts index eecbfec98f..77e61fb089 100644 --- a/src/compat/maestro/__tests__/engine.test.ts +++ b/src/compat/maestro/__tests__/engine.test.ts @@ -128,6 +128,51 @@ describe('executeMaestroProgram', () => { ); }); + test('continues after optional scrollUntilVisible and extendedWaitUntil misses', async () => { + const execute = vi.fn(async (request: MaestroRuntimeRequest) => { + request.invalidateObservation(); + return {}; + }); + const port = makePort({ + observe: vi.fn(async ({ generation }) => ({ generation, matched: false })), + execute, + }); + const program = parseMaestroProgram( + [ + '---', + '- scrollUntilVisible:', + ' element:', + ' id: missing', + ' optional: true', + '- extendedWaitUntil:', + ' visible:', + ' text: Missing', + ' optional: true', + ' timeout: 1', + '- inputText: continued', + ].join('\n'), + ); + execute.mockImplementationOnce(async () => { + throw maestroTestFailure('Maestro scrollUntilVisible target did not become visible.'); + }); + + const result = await executeMaestroProgram(program, port); + + expect(result).toMatchObject({ + executed: 1, + skipped: 2, + }); + expect(result.warnings).toEqual([ + expect.stringMatching(/Optional Maestro scrollUntilVisible skipped at line 2/), + expect.stringMatching(/Optional Maestro extendedWaitUntil skipped at line 6/), + ]); + expect(execute).toHaveBeenCalledWith( + expect.objectContaining({ + command: expect.objectContaining({ kind: 'inputText', text: 'continued' }), + }), + ); + }); + test('propagates AMBIGUOUS_MATCH from an optional target command', async () => { const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple target matches'); const execute = vi.fn(async () => { diff --git a/src/compat/maestro/__tests__/program-ir-parser.test.ts b/src/compat/maestro/__tests__/program-ir-parser.test.ts index 761e4715b6..d52d05792d 100644 --- a/src/compat/maestro/__tests__/program-ir-parser.test.ts +++ b/src/compat/maestro/__tests__/program-ir-parser.test.ts @@ -203,6 +203,83 @@ describe('parseMaestroProgram', () => { }); }); + test('parses optional on scrollUntilVisible and extendedWaitUntil element selectors', () => { + const program = parseMaestroProgram( + [ + '---', + '- scrollUntilVisible:', + ' element:', + ' id: maybe-visible', + ' optional: true', + '- extendedWaitUntil:', + ' visible:', + ' text: Ready', + ' optional: true', + ' timeout: 1000', + '- extendedWaitUntil:', + ' notVisible:', + ' id: gone', + ' optional: true', + ].join('\n'), + ); + + assert.deepEqual(program.commands[0], { + kind: 'scrollUntilVisible', + source: { line: 2 }, + element: { id: 'maybe-visible' }, + optional: true, + }); + assert.deepEqual(program.commands[1], { + kind: 'extendedWaitUntil', + source: { line: 6 }, + visible: { text: 'Ready' }, + timeout: 1000, + optional: true, + }); + assert.deepEqual(program.commands[2], { + kind: 'extendedWaitUntil', + source: { line: 11 }, + notVisible: { id: 'gone' }, + optional: true, + }); + }); + + test('rejects selectors that contain only optional and no matching criteria', () => { + assert.throws( + () => + parseMaestroProgram( + ['---', '- scrollUntilVisible:', ' element:', ' optional: true'].join('\n'), + ), + /scrollUntilVisible\.element selector must contain a selector value/i, + ); + assert.throws( + () => + parseMaestroProgram( + ['---', '- extendedWaitUntil:', ' visible:', ' optional: true'].join('\n'), + ), + /extendedWaitUntil\.visible selector must contain a selector value/i, + ); + assert.throws( + () => + parseMaestroProgram( + ['---', '- extendedWaitUntil:', ' notVisible:', ' optional: true'].join('\n'), + ), + /extendedWaitUntil\.notVisible selector must contain a selector value/i, + ); + }); + + test('rejects extendedWaitUntil with both visible and notVisible conditions', () => { + assert.throws( + () => + parseMaestroProgram( + ['---', '- extendedWaitUntil:', ' visible: A', ' notVisible:', ' id: B'].join( + '\n', + ), + ), + /extendedWaitUntil cannot specify both visible and notVisible/i, + ); + }); + test('preserves an include boundary and the authored include path', () => { const program = parseMaestroProgram( `appId: example.app diff --git a/src/compat/maestro/program-ir-command-parser.ts b/src/compat/maestro/program-ir-command-parser.ts index d0189738e0..385e827f27 100644 --- a/src/compat/maestro/program-ir-command-parser.ts +++ b/src/compat/maestro/program-ir-command-parser.ts @@ -15,6 +15,7 @@ import type { MaestroPressKeyCommand, MaestroScrollCommand, MaestroScrollUntilVisibleCommand, + MaestroSelector, MaestroStopAppCommand, MaestroTakeScreenshotCommand, MaestroWaitForAnimationToEndCommand, @@ -28,6 +29,7 @@ import { parseMaestroSwipeCommand, parseMaestroTapOnCommand, } from './program-ir-gesture-parser.ts'; +import { MAESTRO_BASE_SELECTOR_KEYS } from './selector-vocabulary.ts'; import { parseMaestroRepeatCommand, parseMaestroRetryCommand, @@ -53,6 +55,7 @@ import { readScalarValue, readSequenceItems, sourceAt, + type MaestroMapEntry, type MaestroProgramParseContext, } from './program-ir-values.ts'; @@ -287,6 +290,55 @@ function parseAssertion( }); } +const OPTIONAL_SELECTOR_KEYS = [...MAESTRO_BASE_SELECTOR_KEYS, 'optional'] as const; + +type ParsedOptionalSelector = { + selector: MaestroSelector; + optional: boolean | undefined; +}; + +function parseOptionalSelector( + entries: readonly MaestroMapEntry[], + key: string, + name: string, + context: MaestroProgramParseContext, +): ParsedOptionalSelector | undefined { + if (!hasEntry(entries, key)) return undefined; + const parsed = parseMaestroSelector( + entryValue(entries, key), + name, + context, + OPTIONAL_SELECTOR_KEYS, + ); + const { optional: selectorOptional, ...selector } = parsed; + return { selector, optional: selectorOptional }; +} + +function parseExtendedWaitUntilCondition( + entries: readonly MaestroMapEntry[], + commandNode: Node, + context: MaestroProgramParseContext, +): { key: 'visible' | 'notVisible'; selector: MaestroSelector; optional?: boolean } { + const visible = parseOptionalSelector(entries, 'visible', 'extendedWaitUntil.visible', context); + const notVisible = parseOptionalSelector( + entries, + 'notVisible', + 'extendedWaitUntil.notVisible', + context, + ); + if (visible && notVisible) + invalidAt( + 'Maestro extendedWaitUntil cannot specify both visible and notVisible.', + commandNode, + context, + ); + if (!visible && !notVisible) + invalidAt('Maestro extendedWaitUntil requires visible or notVisible.', commandNode, context); + return visible + ? { key: 'visible', selector: visible.selector, optional: visible.optional } + : { key: 'notVisible', selector: notVisible!.selector, optional: notVisible!.optional }; +} + function parseExtendedWaitUntil( value: Node | null, commandNode: Node, @@ -300,29 +352,19 @@ function parseExtendedWaitUntil( context, ); const options = readOptionalCommandOption(entries, 'extendedWaitUntil', context); - const visible = hasEntry(entries, 'visible') - ? parseMaestroSelector(entryValue(entries, 'visible'), 'extendedWaitUntil.visible', context) - : undefined; - const notVisible = hasEntry(entries, 'notVisible') - ? parseMaestroSelector( - entryValue(entries, 'notVisible'), - 'extendedWaitUntil.notVisible', - context, - ) - : undefined; - if (visible === undefined && notVisible === undefined) - invalidAt('Maestro extendedWaitUntil requires visible or notVisible.', commandNode, context); + const condition = parseExtendedWaitUntilCondition(entries, commandNode, context); const timeout = hasEntry(entries, 'timeout') ? readOptionalNumber(entryValue(entries, 'timeout'), 'extendedWaitUntil.timeout', context) : undefined; - return stripUndefined({ + const optional = options.optional === true || condition.optional === true ? true : undefined; + const command: MaestroExtendedWaitUntilCommand = { kind: 'extendedWaitUntil' as const, source: sourceAt(commandNode, context), - visible, - notVisible, timeout, - ...options, - }); + optional, + }; + command[condition.key] = condition.selector; + return stripUndefined(command); } function parseTakeScreenshot( @@ -367,13 +409,14 @@ function parseScrollUntilVisible( context, ); const options = readOptionalCommandOption(entries, 'scrollUntilVisible', context); - if (!hasEntry(entries, 'element')) - invalidAt('Maestro scrollUntilVisible requires element.', commandNode, context); - const element = parseMaestroSelector( - entryValue(entries, 'element'), + const parsedElement = parseOptionalSelector( + entries, + 'element', 'scrollUntilVisible.element', context, ); + if (!parsedElement) + invalidAt('Maestro scrollUntilVisible requires element.', commandNode, context); const direction = hasEntry(entries, 'direction') ? parseMaestroDirection( entryValue(entries, 'direction'), @@ -384,13 +427,14 @@ function parseScrollUntilVisible( const timeout = hasEntry(entries, 'timeout') ? readOptionalNumber(entryValue(entries, 'timeout'), 'scrollUntilVisible.timeout', context) : undefined; + const optional = options.optional === true || parsedElement!.optional === true ? true : undefined; return stripUndefined({ kind: 'scrollUntilVisible' as const, source, - element, + element: parsedElement!.selector, direction, timeout, - ...options, + optional, }); } diff --git a/src/compat/maestro/program-ir-gesture-parser.ts b/src/compat/maestro/program-ir-gesture-parser.ts index d195ce833f..530e92a85d 100644 --- a/src/compat/maestro/program-ir-gesture-parser.ts +++ b/src/compat/maestro/program-ir-gesture-parser.ts @@ -60,6 +60,8 @@ const SELECTOR_FIELD_READERS: Readonly> = { assignBooleanSelector(selector, 'enabled', entry, name, context), selected: (selector, entry, name, context) => assignBooleanSelector(selector, 'selected', entry, name, context), + optional: (selector, entry, name, context) => + assignBooleanSelector(selector, 'optional', entry, name, context), }; export function parseMaestroSelector( @@ -92,7 +94,8 @@ export function parseMaestroSelectorMapEntries( } read(selector, entry, name, context); } - if (Object.keys(selector).length === 0) { + const matchingKeys = Object.keys(selector).filter((key) => key !== 'optional'); + if (matchingKeys.length === 0) { invalidAt( `Maestro ${name} selector must contain a selector value.`, entries[0]?.keyNode, @@ -411,7 +414,7 @@ function assignStringSelector( function assignBooleanSelector( selector: MaestroSelectorMap, - key: 'enabled' | 'selected', + key: 'enabled' | 'selected' | 'optional', entry: MaestroMapEntry, name: string, context: MaestroProgramParseContext, diff --git a/src/compat/maestro/program-ir.ts b/src/compat/maestro/program-ir.ts index 6adfae8e5b..028f52e758 100644 --- a/src/compat/maestro/program-ir.ts +++ b/src/compat/maestro/program-ir.ts @@ -19,6 +19,7 @@ export type MaestroSelectorMap = { label?: string; enabled?: boolean; selected?: boolean; + optional?: boolean; }; export type MaestroSelector = MaestroSelectorMap; diff --git a/src/compat/maestro/support-matrix.ts b/src/compat/maestro/support-matrix.ts index b2ce0af3c0..4cc0c9e072 100644 --- a/src/compat/maestro/support-matrix.ts +++ b/src/compat/maestro/support-matrix.ts @@ -5,7 +5,7 @@ export const MAESTRO_COMPAT_SUPPORTED_CAPABILITIES = [ 'deterministic repeat.times and retry blocks', 'tapOn including index, childOf, label, and absolute/percentage point taps', 'doubleTapOn and longPressOn', - 'optional target and assertion commands', + 'optional target, assertion, scrollUntilVisible, and extendedWaitUntil commands', 'inputText and focused-field eraseText', 'openLink', 'visibility assertions including childOf and extendedWaitUntil', diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index d026199b3e..8f149cf523 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -62,7 +62,7 @@ Maestro compatibility parses supported YAML into a source-preserving typed progr - Supported and unsupported capabilities: https://github.com/callstack/agent-device/issues/558 - New focused compatibility request: https://github.com/callstack/agent-device/issues/new -Currently supported areas include app launch with Apple-platform launch arguments and Android/iOS simulator `clearState`, `runFlow` file/inline with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` and `onFlowComplete` hooks, deterministic `repeat.times` and retry blocks, `tapOn` including `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn` and `longPressOn`, `optional` target and assertion commands, `inputText` and focused-field `eraseText`, `openLink`, visibility assertions including `childOf` and `extendedWaitUntil`, `scroll` and `scrollUntilVisible`, absolute/percentage `swipe` and `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`, and ordered trusted `runScript` file/env scripts with `http.post`, `json`, and `output` variables. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. +Currently supported areas include app launch with Apple-platform launch arguments and Android/iOS simulator `clearState`, `runFlow` file/inline with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` and `onFlowComplete` hooks, deterministic `repeat.times` and retry blocks, `tapOn` including `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn` and `longPressOn`, `optional` target, assertion, `scrollUntilVisible`, and `extendedWaitUntil` commands, `inputText` and focused-field `eraseText`, `openLink`, visibility assertions including `childOf` and `extendedWaitUntil`, `scroll` and `scrollUntilVisible`, absolute/percentage `swipe` and `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`, and ordered trusted `runScript` file/env scripts with `http.post`, `json`, and `output` variables. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both.