diff --git a/dist/build.js b/dist/build.js index 38c73ca2..5a52ebd6 100644 --- a/dist/build.js +++ b/dist/build.js @@ -58,6 +58,7 @@ const patches = [ "peekaboo-minimized-close-test.patch", "peekaboo-notification-actions.patch", "peekaboo-fullscreen.patch", + "peekaboo-date-value.patch", ].map((name) => join(native, "patches", name)); function spawn(command, env) { return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env }); diff --git a/dist/client.js b/dist/client.js index c32644d6..63eb454d 100644 --- a/dist/client.js +++ b/dist/client.js @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; import { failureAvailability, windowAvailability } from "./availability.js"; +import { canonicalDate } from "./date-value.js"; import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_ELEMENT_ACTIONS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js"; import { NOTIFICATIONS_MS, notificationAction, readNotifications, validateNotificationAction, } from "./notifications.js"; import { waitFor } from "./wait.js"; @@ -674,8 +675,11 @@ function validateAction(request) { throw new Error(`Choose one of ${DESKTOP_ELEMENT_ACTIONS.join(", ")}.`); } if (request.op === "set-value" && typeof request.value !== "boolean" - && (typeof request.value !== "number" || !Number.isFinite(request.value))) { - throw new Error("Set a finite number or true/false; use desktop_type for text."); + && (typeof request.value !== "number" || !Number.isFinite(request.value)) + && (typeof request.value !== "string" || canonicalDate(request.value) === undefined)) { + throw new Error(typeof request.value === "string" + ? "Set a date as ISO 8601: 2026-10-17, 2026-10-17T15:30 (local time) or 2026-10-17T15:30:00Z. Use desktop_type for text." + : "Set a finite number, true/false or an ISO 8601 date; use desktop_type for text."); } if (request.op === "type" && (typeof request.text !== "string" || request.text.length > 8192)) { throw new Error("Replacement text must contain at most 8192 UTF-16 code units."); @@ -876,7 +880,9 @@ async function act(native, request, signal) { format: request.format, image: await clipboardImageInput(request.path, signal), } - : request; + : request.op === "set-value" && typeof request.value === "string" + ? { ...request, value: canonicalDate(request.value) } + : request; const operation = request.op === "menu" ? "menu" : request.op === "dialog-answer" @@ -924,6 +930,12 @@ async function act(native, request, signal) { }, outcome); } const outcome = data.outcome; + if (request.op === "set-value" && typeof request.value === "string" && outcome !== "completed") { + data = { + ...data, + keyboard_route: "If the date control refuses Accessibility, click its first field (such as the month) with desktop_click, type its digits with desktop_key, and press Tab to move to the next field, or Up and Down to step the focused field; then inspect to read the value back.", + }; + } if (request.op === "menu") { const receipt = data.target_receipt; if ((outcome === "completed" || receipt) && (!receipt || receipt.window_id !== undefined diff --git a/dist/date-value.d.ts b/dist/date-value.d.ts new file mode 100644 index 00000000..ebf526fa --- /dev/null +++ b/dist/date-value.d.ts @@ -0,0 +1,5 @@ +/** + * Reads an ISO 8601 date or date-time as the UTC instant a date control stores, to the second. + * A date alone is local midnight and a time without an offset is local, as a date picker shows them. + */ +export declare function canonicalDate(text: string): string | undefined; diff --git a/dist/date-value.js b/dist/date-value.js new file mode 100644 index 00000000..f0c58792 --- /dev/null +++ b/dist/date-value.js @@ -0,0 +1,31 @@ +const pattern = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2}))?(Z|[+-]\d{2}:\d{2})?)?$/; +/** + * Reads an ISO 8601 date or date-time as the UTC instant a date control stores, to the second. + * A date alone is local midnight and a time without an offset is local, as a date picker shows them. + */ +export function canonicalDate(text) { + const match = pattern.exec(text); + if (!match) + return undefined; + const [year, month, day, hour, minute, second] = match.slice(1, 7).map((part) => Number(part ?? 0)); + const zone = match[7]; + if (month < 1 || month > 12 || day < 1 || hour > 23 || minute > 59 || second > 59) + return undefined; + // Rejects days past the month's end, which Date would roll into the next month. + const days = new Date(Date.UTC(year, month, 0)).getUTCDate(); + if (day > days) + return undefined; + let time; + if (zone === undefined) { + time = new Date(year, month - 1, day, hour, minute, second).getTime(); + } + else { + const offset = zone === "Z" + ? 0 + : (zone[0] === "-" ? -1 : 1) * (Number(zone.slice(1, 3)) * 60 + Number(zone.slice(4, 6))); + time = Date.UTC(year, month - 1, day, hour, minute, second) - offset * 60_000; + } + if (!Number.isFinite(time)) + return undefined; + return new Date(time).toISOString().replace(/\.\d{3}Z$/, "Z"); +} diff --git a/dist/protocol.d.ts b/dist/protocol.d.ts index 1d602ed1..ddf59e34 100644 --- a/dist/protocol.d.ts +++ b/dist/protocol.d.ts @@ -226,7 +226,7 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D op: "set-value"; snapshot: string; element: string; - value: number | boolean; + value: number | boolean | string; } | { op: "select"; snapshot: string; diff --git a/dist/tools.js b/dist/tools.js index 89e746f5..657640fb 100644 --- a/dist/tools.js +++ b/dist/tools.js @@ -690,14 +690,14 @@ Optional modifiers (command, control, option and shift, each at most once) are h { name: "desktop_set_value", op: "set-value", - description: `Set the value of one non-text control from a fresh desktop_inspect result and verify it by reading the same element back: a number for a slider, stepper (AXIncrementor), or level indicator, or true/false for a checkbox, switch, or disclosure triangle (true expands it). Pass snapshot_id as snapshot and the literal element ID as element. A number is written directly and must read back exactly, so a control that rounds or clamps it reports unknown; desktop_element_action with AXIncrement or AXDecrement steps a control without a settable value. A toggle with a read-only value that advertises AXPress gets at most one press, and none when it already has the requested state; a mixed state is refused. Use desktop_type for text fields. A pop-up button has no settable value: open it with desktop_click, then choose the item from the fresh observation, which lists an open menu's items first, with desktop_element_action AXPick or desktop_click. The snapshot binds the exact process, window, and element and is single-use. This does not activate the window. Inspect again after an unknown result and never blindly repeat interrupted input.`, + description: `Set the value of one non-text control from a fresh desktop_inspect result and verify it by reading the same element back: a number for a slider, stepper (AXIncrementor), or level indicator; true/false for a checkbox, switch, or disclosure triangle (true expands it); or an ISO 8601 date for a date or time picker (AXDateTimeArea, AXDateField or AXTimeField): 2026-10-17 is local midnight, 2026-10-17T15:30 is local time, and Z or an offset such as +02:00 names the zone. A date is written as the control's date value and must read back to the second; the result shows it in UTC, and a control that drops the time or seconds reports unknown. If a date control refuses, the result's keyboard_route explains entering it field by field. Pass snapshot_id as snapshot and the literal element ID as element. A number is written directly and must read back exactly, so a control that rounds or clamps it reports unknown; desktop_element_action with AXIncrement or AXDecrement steps a control without a settable value. A toggle with a read-only value that advertises AXPress gets at most one press, and none when it already has the requested state; a mixed state is refused. Use desktop_type for text fields. A pop-up button has no settable value: open it with desktop_click, then choose the item from the fresh observation, which lists an open menu's items first, with desktop_element_action AXPick or desktop_click. The snapshot binds the exact process, window, and element and is single-use. This does not activate the window. Inspect again after an unknown result and never blindly repeat interrupted input.`, parameters: { type: "object", required: ["snapshot", "element", "value"], properties: { snapshot: id, element: id, - value: { anyOf: [{ type: "number" }, { type: "boolean" }] }, + value: { anyOf: [{ type: "number" }, { type: "boolean" }, { type: "string", maxLength: 64 }] }, }, }, }, diff --git a/docs/tools.md b/docs/tools.md index c7913e39..9db4cf9d 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -103,7 +103,7 @@ A status item is pressed through Accessibility in its owner process without movi | `desktop_press` | Press and hold an element or point for up to 10 seconds, then release in place | | `desktop_type` | Replace an editable field's whole value | | `desktop_select` | Select literal text in a field, or put the caret before or after it | -| `desktop_set_value` | Set a slider, stepper or level indicator to a number, or a checkbox, switch or disclosure triangle on or off, and read it back | +| `desktop_set_value` | Set a slider, stepper or level indicator to a number, a checkbox, switch or disclosure triangle on or off, or a date or time picker to an ISO 8601 date, and read it back | | `desktop_element_action` | Perform one Accessibility action an element lists, such as `AXShowMenu`, `AXIncrement`, `AXDecrement`, `AXPick` or `AXCancel` | | `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard; where no text control can be observed, type it into the window's keyboard focus | | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key | diff --git a/native/patches/README.md b/native/patches/README.md index b2f5e15d..44bd03b4 100644 --- a/native/patches/README.md +++ b/native/patches/README.md @@ -459,6 +459,18 @@ exact window, and a signed confirmed response must carry the requested state. Te rules and the signed postcondition for both states; the operation inventory counts rise to 128. +`peekaboo-date-value.patch` lets set-value write a date. Date and time controls such as `NSDatePicker` expose an +`NSDate` as `AXValue`, which upstream `setValue` cannot read back (`ElementValueReadback` has no date) and +would overwrite with a string. With the patch, when the element's current value is a date and the request is +a string, the driver takes a date path: the string must be UTC ISO 8601 to the second +(`2026-10-17T15:30:00Z`, the one spelling `ElementDateValue` formats), anything else is refused as +`valueNotSettable` before dispatch, and an equal current value completes with no change. Otherwise it writes +an `NSDate` through the new `setAutomationDate`, polls the value for up to one second, and confirms only when +it reads back the same string; else the outcome is the usual unverified value mutation. The verification +reports the readback as a string in the same spelling, so the existing signed set-value binding, which +compares the readback with the requested string, applies unchanged. Other values and elements take the +existing path. Tests cover the write, the no-change case, refused spellings and a mismatched readback. + When updating Peekaboo, review the upstream fixes and these patches together. Update the revision guard and patches deliberately, or remove each patch when the dependency includes its fix. Revalidate real button clicks, changed-focus refusal, literal multiline insertion, clipboard diff --git a/native/patches/peekaboo-date-value.patch b/native/patches/peekaboo-date-value.patch new file mode 100644 index 00000000..f3df80a0 --- /dev/null +++ b/native/patches/peekaboo-date-value.patch @@ -0,0 +1,214 @@ +diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift +index b09931f88..65f3ea3a3 100644 +--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift ++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ActionInputDriver.swift +@@ -796,6 +796,11 @@ struct ActionInputDriver: ActionInputDriving { + attribute: .selected, resolvedKind: .bool, readback: .bool(selectedAfter))) + } + ++ if let dateBefore = element.value as? Date, case let .string(text) = value { ++ return try await self.setDateValue( ++ text, before: dateBefore, on: element, beforeMutation: beforeMutation) ++ } ++ + let valueBefore = element.value + let readbackBefore = ElementValueReadback(nativeValue: valueBefore) + guard !(valueBefore is NSNumber) || readbackBefore != nil else { +@@ -868,6 +873,47 @@ struct ActionInputDriver: ActionInputDriving { + } + } + ++ /// Date and time controls report an NSDate; the request and readback are both UTC ISO 8601 to the second. ++ private func setDateValue( ++ _ text: String, ++ before: Date, ++ on element: any AutomationElementRepresenting, ++ beforeMutation: @MainActor () throws -> Void) async throws -> UIInputExecutionResult.Action ++ { ++ guard let requested = ElementDateValue.date(text) else { ++ throw ActionInputError.unsupported(.valueNotSettable) ++ } ++ func verification(_ date: Date) -> ElementValueVerification { ++ .init(attribute: .value, resolvedKind: .string, readback: .string(ElementDateValue.string(date))) ++ } ++ if ElementDateValue.string(before) == text { ++ return UIInputExecutionResult.Action( ++ outcome: .confirmedNoChange(), ++ actionName: AXActionNames.kAXSetValueAction, ++ anchorPoint: element.anchorPoint, ++ elementRole: element.role, ++ valueVerification: verification(before)) ++ } ++ try Self.validateBeforeMutation(beforeMutation) ++ try element.setAutomationDate(requested) ++ // Controls apply the date synchronously, but may publish it on the next run-loop turn. ++ let deadline = ContinuousClock.now.advanced(by: .seconds(1)) ++ while true { ++ if let after = element.value as? Date, ElementDateValue.string(after) == text { ++ return UIInputExecutionResult.Action( ++ outcome: Self.dispatchedValueMutationOutcome(preStateKnown: true), ++ actionName: AXActionNames.kAXSetValueAction, ++ anchorPoint: element.anchorPoint, ++ elementRole: element.role, ++ valueVerification: verification(after)) ++ } ++ guard ContinuousClock.now < deadline else { ++ throw Self.unverifiedValueMutationFailure(attribute: AXActionNames.kAXSetValueAction) ++ } ++ try await Task.sleep(for: .milliseconds(50)) ++ } ++ } ++ + func performObservedMutation( + on element: any AutomationElementRepresenting, + attribute: AXMutationObservationAttribute, +diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/AutomationElement.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/AutomationElement.swift +index 9adc55341..eabb47429 100644 +--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/AutomationElement.swift ++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/AutomationElement.swift +@@ -50,6 +50,8 @@ protocol AutomationElementRepresenting: Sendable { + + func performAutomationAction(_ actionName: String) throws + func setAutomationValue(_ value: UIElementValue) throws ++ /// Writes an NSDate as AXValue, as date and time controls take it. ++ func setAutomationDate(_ date: Date) throws + func setAutomationFocused(_ focused: Bool) throws + /// Writes AXFocused=true and returns only when the write does, without occupying MainActor meanwhile. + func focusAutomationElement() async throws +@@ -93,6 +95,10 @@ extension AutomationElementRepresenting { + nil + } + ++ func setAutomationDate(_: Date) throws { ++ throw ActionInputError.unsupported(.valueNotSettable) ++ } ++ + func supportsAction(_ actionName: String) -> Bool { + self.actionNames.contains(actionName) + } +@@ -340,6 +346,17 @@ struct AutomationElement: AutomationElementRepresenting { + } + } + ++ @MainActor ++ func setAutomationDate(_ date: Date) throws { ++ let error = AXUIElementSetAttributeValue( ++ self.element.underlyingElement, ++ AXAttributeNames.kAXValueAttribute as CFString, ++ date as NSDate) ++ guard error == .success else { ++ throw AccessibilitySystemError(error) ++ } ++ } ++ + @MainActor + func setAutomationFocused(_ focused: Bool) throws { + let error = AXUIElementSetAttributeValue( +diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ElementDateValue.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ElementDateValue.swift +new file mode 100644 +index 000000000..0a4a39a27 +--- /dev/null ++++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/UI/ElementDateValue.swift +@@ -0,0 +1,21 @@ ++import Foundation ++ ++/// Date and time controls such as NSDatePicker expose an NSDate as AXValue. Requests and readbacks spell ++/// it as UTC ISO 8601 to the second, so a readback string matches its request only when the dates agree. ++enum ElementDateValue { ++ static func date(_ text: String) -> Date? { ++ guard let date = self.formatter.date(from: text), self.string(date) == text else { return nil } ++ return date ++ } ++ ++ static func string(_ date: Date) -> String { ++ self.formatter.string(from: Date(timeIntervalSince1970: date.timeIntervalSince1970.rounded(.down))) ++ } ++ ++ private static var formatter: ISO8601DateFormatter { ++ let formatter = ISO8601DateFormatter() ++ formatter.formatOptions = [.withInternetDateTime] ++ formatter.timeZone = TimeZone(secondsFromGMT: 0) ++ return formatter ++ } ++} +diff --git a/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTestFixtures.swift b/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTestFixtures.swift +index 8d82ed53a..b704f7dd2 100644 +--- a/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTestFixtures.swift ++++ b/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTestFixtures.swift +@@ -444,6 +444,17 @@ final class ActionInputMockAutomationElement: AutomationElementRepresenting, @un + } + } + ++ var dateSetterReadbackOverride: Date? ++ var setDates: [Date] = [] ++ ++ func setAutomationDate(_ date: Date) throws { ++ guard self.isValueSettable else { ++ throw AccessibilitySystemError(.attributeUnsupported) ++ } ++ self.setDates.append(date) ++ self.value = self.dateSetterReadbackOverride ?? date ++ } ++ + func setAutomationFocused(_ focused: Bool) throws { + guard self.isFocusedSettable else { + throw AccessibilitySystemError(.attributeUnsupported) +diff --git a/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTests.swift b/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTests.swift +index 9363c5f07..449aee1c7 100644 +--- a/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTests.swift ++++ b/Core/PeekabooAutomationKit/Tests/PeekabooAutomationKitTests/ActionInputDriverTests.swift +@@ -802,6 +802,54 @@ extension ActionInputDriverTests { + #expect(result.actionName == AXActionNames.kAXSetValueAction) + } + ++ @MainActor ++ @Test ++ func `date value is written as a date and verified by its UTC readback`() async throws { ++ let element = ActionInputMockAutomationElement( ++ role: "AXDateTimeArea", ++ value: Date(timeIntervalSince1970: 1_700_000_000.5), ++ isValueSettable: true) ++ ++ let result = try await ActionInputDriver().trySetValueForTesting( ++ element: element, value: .string("2026-10-17T15:30:00Z")) ++ ++ #expect(element.setDates == [Date(timeIntervalSince1970: 1_792_251_000)]) ++ #expect(element.setValues.isEmpty) ++ #expect(result.outcome.state == .confirmedChange) ++ #expect(result.valueVerification?.readback == .string("2026-10-17T15:30:00Z")) ++ #expect(result.valueVerification?.matches( ++ requested: .string("2026-10-17T15:30:00Z"), ++ newValue: "2026-10-17T15:30:00Z", ++ actionName: AXActionNames.kAXSetValueAction) == true) ++ ++ let same = try await ActionInputDriver().trySetValueForTesting( ++ element: element, value: .string("2026-10-17T15:30:00Z")) ++ #expect(same.outcome.state == .confirmedNoChange) ++ #expect(element.setDates.count == 1) ++ } ++ ++ @MainActor ++ @Test ++ func `date value refuses non canonical text and reports a mismatched readback as unknown`() async throws { ++ let element = ActionInputMockAutomationElement( ++ role: "AXDateTimeArea", ++ value: Date(timeIntervalSince1970: 1_700_000_000), ++ isValueSettable: true) ++ for text in ["2026-10-17", "2026-10-17T15:30:00+02:00", "2026-10-17T15:30:00.250Z", "tomorrow"] { ++ await #expect(throws: ActionInputError.unsupported(.valueNotSettable)) { ++ _ = try await ActionInputDriver().trySetValueForTesting(element: element, value: .string(text)) ++ } ++ } ++ #expect(element.setDates.isEmpty) ++ ++ element.dateSetterReadbackOverride = Date(timeIntervalSince1970: 1_792_195_200) ++ await #expect(throws: DesktopActionFailure.self) { ++ _ = try await ActionInputDriver().trySetValueForTesting( ++ element: element, value: .string("2026-10-17T15:30:00Z")) ++ } ++ #expect(element.setDates.count == 1) ++ } ++ + @MainActor + @Test + func `boolean selected attribute is set and verified`() async throws { diff --git a/native/sources/client/actions.swift b/native/sources/client/actions.swift index 771dde91..66292d5a 100644 --- a/native/sources/client/actions.swift +++ b/native/sources/client/actions.swift @@ -253,10 +253,14 @@ func customActionName(_ action: String) -> String? { private let booleanValueRoles: Set = ["AXCheckBox", "AXSwitch", "AXToggle", "AXDisclosureTriangle"] /// Roles whose Accessibility value is one number. private let numericValueRoles: Set = ["AXSlider", "AXIncrementor", "AXStepper", "AXLevelIndicator"] +/// Roles whose Accessibility value is one date, which the host writes as an NSDate. +private let dateValueRoles: Set = ["AXDateTimeArea", "AXDateField", "AXTimeField"] private let textValueRoles: Set = [ "AXTextField", "AXTextArea", "AXComboBox", "AXSearchField", "AXSecureTextField", ] +let dateKeyboardRoute = "To enter the date from the keyboard instead, click the control's first field (such as the month) with desktop_click, type its digits with desktop_key, and press Tab to move to the next field, or Up and Down to step the focused field; then inspect to read the value back." + private let pointerClicks: Set = ["single", "double", "right", "middle", "triple"] let semanticShortcuts = [ "w": "desktop_close", "q": "desktop_quit", "h": "desktop_hide", "m": "desktop_minimize", @@ -846,6 +850,18 @@ private func checkElementRequest(_ request: ActionRequest, _ observed: DetectedE if textValueRoles.contains(role) { throw ActionError("Use desktop_type to replace a text control's value.") } + if dateValueRoles.contains(role) { + guard case .string = request.value! else { + throw ActionError("A \(role) takes an ISO 8601 date, such as 2026-10-17T15:30.") + } + guard observed.attributes["isValueSettable"] != "false" else { + throw ActionError("This \(role) has no settable value. " + dateKeyboardRoute) + } + return + } + if case .string = request.value! { + throw ActionError("Only date and time controls take a date; this element is \(role.isEmpty ? "unknown" : role).") + } let boolean = booleanValueRoles.contains(role) guard boolean || numericValueRoles.contains(role) else { throw ActionError("Set values only on sliders, steppers, level indicators, checkboxes, switches, and disclosure triangles; this element is \(role.isEmpty ? "unknown" : role). Choose a pop-up button's item by clicking the button and then the item.") diff --git a/src/build.ts b/src/build.ts index 5532f392..91d0c4b2 100644 --- a/src/build.ts +++ b/src/build.ts @@ -59,6 +59,7 @@ const patches = [ "peekaboo-minimized-close-test.patch", "peekaboo-notification-actions.patch", "peekaboo-fullscreen.patch", + "peekaboo-date-value.patch", ].map((name) => join(native, "patches", name)); function spawn(command: string[], env?: NodeJS.ProcessEnv) { diff --git a/src/client.ts b/src/client.ts index 6e2dc6cf..ebce7019 100644 --- a/src/client.ts +++ b/src/client.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; import { failureAvailability, windowAvailability } from "./availability.js"; +import { canonicalDate } from "./date-value.js"; import { DESKTOP_BUTTONS, @@ -807,8 +808,13 @@ function validateAction(request: DesktopAction) { if ( request.op === "set-value" && typeof request.value !== "boolean" && (typeof request.value !== "number" || !Number.isFinite(request.value)) + && (typeof request.value !== "string" || canonicalDate(request.value) === undefined) ) { - throw new Error("Set a finite number or true/false; use desktop_type for text."); + throw new Error( + typeof request.value === "string" + ? "Set a date as ISO 8601: 2026-10-17, 2026-10-17T15:30 (local time) or 2026-10-17T15:30:00Z. Use desktop_type for text." + : "Set a finite number, true/false or an ISO 8601 date; use desktop_type for text.", + ); } if (request.op === "type" && (typeof request.text !== "string" || request.text.length > 8192)) { throw new Error("Replacement text must contain at most 8192 UTF-16 code units."); @@ -1053,6 +1059,8 @@ async function act( format: request.format, image: await clipboardImageInput(request.path, signal), } + : request.op === "set-value" && typeof request.value === "string" + ? { ...request, value: canonicalDate(request.value) } : request; const operation = request.op === "menu" ? "menu" @@ -1100,6 +1108,13 @@ async function act( }, outcome); } const outcome = data.outcome as DesktopOutcome; + if (request.op === "set-value" && typeof request.value === "string" && outcome !== "completed") { + data = { + ...data, + keyboard_route: + "If the date control refuses Accessibility, click its first field (such as the month) with desktop_click, type its digits with desktop_key, and press Tab to move to the next field, or Up and Down to step the focused field; then inspect to read the value back.", + }; + } if (request.op === "menu") { const receipt = data.target_receipt as Reply["target_receipt"]; if ( diff --git a/src/date-value.ts b/src/date-value.ts new file mode 100644 index 00000000..b3ab4877 --- /dev/null +++ b/src/date-value.ts @@ -0,0 +1,27 @@ +const pattern = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2}))?(Z|[+-]\d{2}:\d{2})?)?$/; + +/** + * Reads an ISO 8601 date or date-time as the UTC instant a date control stores, to the second. + * A date alone is local midnight and a time without an offset is local, as a date picker shows them. + */ +export function canonicalDate(text: string): string | undefined { + const match = pattern.exec(text); + if (!match) return undefined; + const [year, month, day, hour, minute, second] = match.slice(1, 7).map((part) => Number(part ?? 0)); + const zone = match[7]; + if (month < 1 || month > 12 || day < 1 || hour > 23 || minute > 59 || second > 59) return undefined; + // Rejects days past the month's end, which Date would roll into the next month. + const days = new Date(Date.UTC(year, month, 0)).getUTCDate(); + if (day > days) return undefined; + let time: number; + if (zone === undefined) { + time = new Date(year, month - 1, day, hour, minute, second).getTime(); + } else { + const offset = zone === "Z" + ? 0 + : (zone[0] === "-" ? -1 : 1) * (Number(zone.slice(1, 3)) * 60 + Number(zone.slice(4, 6))); + time = Date.UTC(year, month - 1, day, hour, minute, second) - offset * 60_000; + } + if (!Number.isFinite(time)) return undefined; + return new Date(time).toISOString().replace(/\.\d{3}Z$/, "Z"); +} diff --git a/src/protocol.ts b/src/protocol.ts index b8f24a81..515cfa7b 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -139,7 +139,7 @@ export type DesktopRequest = | { op: "type"; snapshot: string; element: string; text: string } | { op: "insert"; snapshot: string; text: string; route?: "paste" | "type" } | { op: "element-action"; snapshot: string; element: string; action: DesktopElementAction } - | { op: "set-value"; snapshot: string; element: string; value: number | boolean } + | { op: "set-value"; snapshot: string; element: string; value: number | boolean | string } | { op: "select"; snapshot: string; diff --git a/src/tools.ts b/src/tools.ts index 0f1f274c..acbc2829 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -742,14 +742,14 @@ Optional modifiers (command, control, option and shift, each at most once) are h name: "desktop_set_value", op: "set-value", description: - `Set the value of one non-text control from a fresh desktop_inspect result and verify it by reading the same element back: a number for a slider, stepper (AXIncrementor), or level indicator, or true/false for a checkbox, switch, or disclosure triangle (true expands it). Pass snapshot_id as snapshot and the literal element ID as element. A number is written directly and must read back exactly, so a control that rounds or clamps it reports unknown; desktop_element_action with AXIncrement or AXDecrement steps a control without a settable value. A toggle with a read-only value that advertises AXPress gets at most one press, and none when it already has the requested state; a mixed state is refused. Use desktop_type for text fields. A pop-up button has no settable value: open it with desktop_click, then choose the item from the fresh observation, which lists an open menu's items first, with desktop_element_action AXPick or desktop_click. The snapshot binds the exact process, window, and element and is single-use. This does not activate the window. Inspect again after an unknown result and never blindly repeat interrupted input.`, + `Set the value of one non-text control from a fresh desktop_inspect result and verify it by reading the same element back: a number for a slider, stepper (AXIncrementor), or level indicator; true/false for a checkbox, switch, or disclosure triangle (true expands it); or an ISO 8601 date for a date or time picker (AXDateTimeArea, AXDateField or AXTimeField): 2026-10-17 is local midnight, 2026-10-17T15:30 is local time, and Z or an offset such as +02:00 names the zone. A date is written as the control's date value and must read back to the second; the result shows it in UTC, and a control that drops the time or seconds reports unknown. If a date control refuses, the result's keyboard_route explains entering it field by field. Pass snapshot_id as snapshot and the literal element ID as element. A number is written directly and must read back exactly, so a control that rounds or clamps it reports unknown; desktop_element_action with AXIncrement or AXDecrement steps a control without a settable value. A toggle with a read-only value that advertises AXPress gets at most one press, and none when it already has the requested state; a mixed state is refused. Use desktop_type for text fields. A pop-up button has no settable value: open it with desktop_click, then choose the item from the fresh observation, which lists an open menu's items first, with desktop_element_action AXPick or desktop_click. The snapshot binds the exact process, window, and element and is single-use. This does not activate the window. Inspect again after an unknown result and never blindly repeat interrupted input.`, parameters: { type: "object", required: ["snapshot", "element", "value"], properties: { snapshot: id, element: id, - value: { anyOf: [{ type: "number" }, { type: "boolean" }] }, + value: { anyOf: [{ type: "number" }, { type: "boolean" }, { type: "string", maxLength: 64 }] }, }, }, }, diff --git a/test/date-value.test.mjs b/test/date-value.test.mjs new file mode 100644 index 00000000..83283c92 --- /dev/null +++ b/test/date-value.test.mjs @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { canonicalDate } from "../dist/date-value.js"; + +test("dates with an offset become UTC to the second", () => { + assert.equal(canonicalDate("2026-10-17T15:30:00Z"), "2026-10-17T15:30:00Z"); + assert.equal(canonicalDate("2026-10-17T15:30+02:00"), "2026-10-17T13:30:00Z"); + assert.equal(canonicalDate("2026-10-17T01:15:05-04:30"), "2026-10-17T05:45:05Z"); +}); + +test("dates without an offset are local", () => { + assert.equal(canonicalDate("2026-10-17"), new Date(2026, 9, 17).toISOString().replace(/\.\d{3}Z$/, "Z")); + assert.equal( + canonicalDate("2026-10-17T09:05"), + new Date(2026, 9, 17, 9, 5).toISOString().replace(/\.\d{3}Z$/, "Z"), + ); +}); + +test("malformed and impossible dates are refused", () => { + for (const text of ["", "tomorrow", "2026-02-30", "2026-13-01", "2026-10-17T24:00", "2026-10-17 15:30", "17/10/2026", "2026-10-17T15:30:00.5Z"]) { + assert.equal(canonicalDate(text), undefined, text); + } +});