diff --git a/apps/desktop/native/sources/client/actions.swift b/apps/desktop/native/sources/client/actions.swift index 9fe40f7a..af10aeab 100644 --- a/apps/desktop/native/sources/client/actions.swift +++ b/apps/desktop/native/sources/client/actions.swift @@ -6,7 +6,7 @@ import PeekabooFoundation private struct ActionRequest: Decodable { enum Operation: String, Decodable { - case click, type, key, select + case click, type, key, select, scroll } let op: Operation @@ -20,6 +20,8 @@ private struct ActionRequest: Decodable { let selection: String? let point: PointerPoint? let kind: String? + let direction: String? + let amount: Int? func validate() throws { guard !snapshot.isEmpty, snapshot.utf16.count <= 256 else { @@ -31,11 +33,17 @@ private struct ActionRequest: Decodable { if op != .select, prefix != nil || suffix != nil || selection != nil { throw ActionError("Only text selection accepts prefix, suffix, or selection.") } - if op != .click, kind != nil || point != nil { - throw ActionError("Only clicks accept a click kind or target point.") + if op != .click, kind != nil { + throw ActionError("Only clicks accept a click kind.") + } + if op != .scroll, direction != nil || amount != nil { + throw ActionError("Only scrolling accepts direction or amount.") + } + if op != .click, op != .scroll, point != nil { + throw ActionError("Only clicks or scrolling accept a target point.") } switch op { - case .click: + case .click, .scroll: guard (element == nil) != (point == nil), text == nil, key == nil else { throw ActionError("Choose exactly one observed element ID or normalized screenshot point.") } @@ -43,8 +51,14 @@ private struct ActionRequest: Decodable { throw ActionError("Choose a literal element ID from the inspected snapshot.") } try point?.validate() - guard kind == nil || pointerClicks.contains(kind!) else { - throw ActionError("Choose single, double, right, middle, or triple click.") + if op == .click { + guard kind == nil || pointerClicks.contains(kind!) else { + throw ActionError("Choose single, double, right, middle, or triple click.") + } + } else { + guard let direction, ScrollDirection(rawValue: direction) != nil, + let amount, (1...20).contains(amount) + else { throw ActionError("Choose up, down, left, or right and 1 to 20 native scroll units.") } } case .type, .select: guard let element, !element.isEmpty, element.utf16.count <= 256, key == nil else { @@ -193,9 +207,14 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data { if request.op == .key, context.focusedElement == nil { throw ActionError("The snapshot has no exact focused control. Click a control, then inspect the window again.") } + let window = try UIAutomationTarget.ExactWindow(identity: identity, bounds: bounds) + var scrollWindow: UIAutomationTarget.ExactWindow? + if request.op == .scroll { + // Request-pinned scroll receipts retain the snapshot's focus evidence as well as its geometry. + scrollWindow = try .init(identity: identity, bounds: bounds, focusedElement: context.focusedElement) + } var point: CGPoint? if request.point != nil { - let window = try UIAutomationTarget.ExactWindow(identity: identity, bounds: bounds) // Normalized coordinates survive host image resizing; authority stays in the bridge's capture. let authority = try SnapshotTargetReceiptPlanner.assemble( snapshotID: request.snapshot, detectionResult: detection @@ -204,6 +223,7 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data { window.bounds.contains(captured), !detection.screenshotPath.isEmpty else { throw ActionError("This observation has no pixel-backed coordinate authority for its exact window.") } point = try request.point?.mapped(in: authority) + } lease = try await client.beginSnapshotMutation(snapshotId: request.snapshot) invoked = true @@ -217,6 +237,12 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data { windowEvidence: .init(identity: identity, bounds: bounds), allowsAccessibilityValueDelivery: true )) + case .scroll: + evidence = try await ActionEvidence(client.scrollWithOutcome(.init( + direction: ScrollDirection(rawValue: request.direction!)!, amount: request.amount!, + target: request.element, point: point, snapshotId: request.snapshot, + expectedWindow: scrollWindow!, foreground: false + ))) case .type: evidence = try await ActionEvidence(client.setValueWithOutcome( target: request.element!, value: .string(request.text!), snapshotId: request.snapshot diff --git a/apps/desktop/native/sources/desktop/desktop.swift b/apps/desktop/native/sources/desktop/desktop.swift index 51e949ef..91e04529 100644 --- a/apps/desktop/native/sources/desktop/desktop.swift +++ b/apps/desktop/native/sources/desktop/desktop.swift @@ -67,7 +67,7 @@ private final class Desktop { .listApplications, .listWindows, .desktopObservation, .ownsSnapshot, .getDetectionResult, .beginSnapshotMutation, .finishSnapshotMutation, .targetedClick, .exactWindowTargetedClick, .setValue, .exactWindowTargetedHotkey, - .selectText, + .selectText, .targetedScroll, .activateApplication, .focusWindow, .restoreWindow, ], hostKind: .gui, diff --git a/apps/host/src/desktop.ts b/apps/host/src/desktop.ts index b18dfebd..d081a620 100644 --- a/apps/host/src/desktop.ts +++ b/apps/host/src/desktop.ts @@ -8,6 +8,7 @@ import { promisify } from "node:util"; import { type Desktop, DESKTOP_CLICKS, + DESKTOP_DIRECTIONS, DESKTOP_KEYS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, @@ -318,7 +319,7 @@ function validateAction(request: DesktopAction) { if (typeof request.snapshot !== "string" || !request.snapshot || request.snapshot.length > 256) { throw new Error("Use the snapshot_id from a fresh desktop_inspect result."); } - if (request.op === "click") { + if (request.op === "click" || request.op === "scroll") { if ((request.element === undefined) === (request.point === undefined)) { throw new Error("Choose exactly one observed element ID or normalized screenshot point."); } @@ -328,8 +329,15 @@ function validateAction(request: DesktopAction) { ) { throw new Error("Choose an element ID from the inspected snapshot."); } - if (request.kind !== undefined && !DESKTOP_CLICKS.includes(request.kind)) { - throw new Error("Choose single, double, right, middle, or triple click."); + if (request.op === "click") { + if (request.kind !== undefined && !DESKTOP_CLICKS.includes(request.kind)) { + throw new Error("Choose single, double, right, middle, or triple click."); + } + } else if ( + !DESKTOP_DIRECTIONS.includes(request.direction) || !Number.isInteger(request.amount) + || request.amount < 1 || request.amount > 20 + ) { + throw new Error("Choose up, down, left, or right and 1 to 20 native scroll units."); } return; } @@ -594,6 +602,7 @@ export const desktop: Desktop = async (request, context) => { "type", "key", "select", + "scroll", "activate", "focus", "restore", diff --git a/docs/architecture.md b/docs/architecture.md index 94910b62..09ab466b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -178,7 +178,7 @@ Peekaboo's native bridge inside Ace's UI process, which owns macOS Accessibility Recording, and Event Synthesizing permissions. The host invokes a bundled client over a local Unix socket. The bridge accepts only the client's exact identifier signed by Ace's team; the client verifies the host's signing team. Tools expose application and window inventories, observation, element or screenshot -clicks, replacement of editable field values, text selection, and keys or shortcuts. Screenshot +clicks, scrolling, replacement of editable field values, text selection, and keys or shortcuts. Screenshot points are normalized and mapped through the snapshot's native capture geometry. Control input uses background delivery bound to a snapshot's exact process, window, and controls; every dispatched action requires a new observation. Separate activation, focus, and restore tools accept generation-bound inventory targets, with exact diff --git a/docs/desktop-tools.md b/docs/desktop-tools.md index 92887b65..44dfa4ff 100644 --- a/docs/desktop-tools.md +++ b/docs/desktop-tools.md @@ -1,7 +1,7 @@ # Native desktop tools Ace's pi harness can inspect applications and windows, activate apps, focus or restore windows, click observed controls, replace editable -field values, select text, and send keys or shortcuts on the machine running the channel's tools. It uses +field values, select text, send keys or shortcuts, and scroll on the machine running the channel's tools. It uses [Peekaboo](https://github.com/openclaw/Peekaboo) for macOS Accessibility, screen capture, and targeted input. The model receives the accessibility text, screenshot, and action outcome. The same result appears in the chat's expandable tool output, including after reopening the channel. @@ -45,6 +45,9 @@ find the client alongside Ace Helper automatically. snapshot or element IDs. The default mode is `accessibility`. - `desktop_click` clicks one observed Accessibility element or screenshot point. `kind` defaults to `single`; `double`, `right`, `middle`, and `triple` are also supported. +- `desktop_scroll` scrolls an observed element or screenshot point `up`, `down`, `left`, or `right`. + `amount` is 1 to 20 native units: Accessibility pages/actions or window-routed wheel ticks, depending + on the target. It is not a pixel distance; inspect the resulting position before continuing. - `desktop_type` replaces the entire string value of one observed editable Accessibility element. It does not append text, send keystrokes, or use the clipboard. Fields that do not support this operation are refused. @@ -80,7 +83,7 @@ application/window inventory even when no inspectable window exists; it never ch window. Inspection stays passive and never activates or restores a target automatically. Inspect the window before clicking or entering input. Pass its `snapshot_id` as `snapshot`. Type and select take the -literal `element` ID from that observation. Click takes exactly one `element` or `point`. +literal `element` ID from that observation. Click and scroll take exactly one `element` or `point`. Points use normalized image coordinates: `x` is the fraction from the screenshot's left edge, `y` from its top edge, each at least 0 and less than 1. For example, `{ "x": 0.5, "y": 0.5 }` is its center. Normalized points retain their meaning when Ace resizes the screenshot. The native @@ -101,7 +104,7 @@ Control and keyboard tools use targeted background delivery. They do not activat mouse or keyboard input when a background route is unavailable. Pointer actions do not move the physical pointer. The target app can still respond by changing its own state or opening a window. Modifier-clicks and long presses need a separate foreground interaction contract; they are not -emulated with held keys or mouse buttons across calls. Literal insertion, scrolling, drag-and-drop, +emulated with held keys or mouse buttons across calls. Literal insertion, drag-and-drop, clipboard operations, foreground interaction, application launch, window geometry/close, menus, and dialogs remain later slices of [native computer use](https://github.com/githubnext/ace2/issues/8). diff --git a/packages/channel/src/desktop.ts b/packages/channel/src/desktop.ts index 01686c4e..8d3b3496 100644 --- a/packages/channel/src/desktop.ts +++ b/packages/channel/src/desktop.ts @@ -35,6 +35,14 @@ export type DesktopRequest = point?: DesktopPoint; kind?: DesktopClick; } + | { + op: "scroll"; + snapshot: string; + element?: string; + point?: DesktopPoint; + direction: DesktopDirection; + amount: number; + } | { op: "type"; snapshot: string; element: string; text: string } | { op: "select"; @@ -49,7 +57,9 @@ export type DesktopRequest = export type DesktopPoint = { x: number; y: number }; export const DESKTOP_CLICKS = ["single", "double", "right", "middle", "triple"] as const; +export const DESKTOP_DIRECTIONS = ["up", "down", "left", "right"] as const; export type DesktopClick = (typeof DESKTOP_CLICKS)[number]; +export type DesktopDirection = (typeof DESKTOP_DIRECTIONS)[number]; export const DESKTOP_KEYS = [ "enter", @@ -122,7 +132,7 @@ export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number]; export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number]; export type DesktopAction = Extract< DesktopRequest, - { op: "click" | "type" | "key" | "select" | "activate" | "focus" | "restore" } + { op: "click" | "type" | "key" | "select" | "scroll" | "activate" | "focus" | "restore" } >; export type DesktopOutcome = "completed" | "refused" | "unknown"; export type DesktopResult = { @@ -135,7 +145,7 @@ export type Desktop = (request: DesktopRequest, context: Context) => Promise act({ op: "click", ...args }, api, context), }), + defineTool({ + name: "desktop_scroll", + description: + "Scroll an observed element or screenshot point in a fresh desktop_inspect result. Pass snapshot_id as snapshot and exactly one literal element ID or normalized point (x from the screenshot's left edge, y from its top edge, each >= 0 and < 1). direction is up, down, left, or right; amount is 1 to 20 native scroll units, whose distance depends on the target's supported route rather than pixels. Uses exact-window background delivery without moving the physical pointer. Unsupported targets are refused. The snapshot is single-use; inspect the resulting position before scrolling again, and never blindly repeat interrupted input.", + parameters: Type.Object({ + snapshot, + element: Type.Optional(element), + point: Type.Optional(point), + direction: Type.Union(DESKTOP_DIRECTIONS.map((value) => Type.Literal(value))), + amount: Type.Integer({ minimum: 1, maximum: 20 }), + }), + replay: "unsafe", + executionMode: "sequential", + execute: async (args, api, context) => act({ op: "scroll", ...args }, api, context), + }), defineTool({ name: "desktop_type", description: