feat: press media, volume and brightness keys as system-defined events

dist/build.js

@@ -60,6 +60,7 @@ const patches = [
     "peekaboo-fullscreen.patch",
     "peekaboo-date-value.patch",
     "peekaboo-web-text-value.patch",
+    "peekaboo-system-defined-key.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.js

@@ -7,7 +7,7 @@ import { promisify } from "node:util";
 import { failureAvailability, windowAvailability } from "./availability.js";
 import { canonicalDate } from "./date-value.js";
 import { saveCapture, validateSave } from "./capture-file.js";
-import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_ELEMENT_ACTIONS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js";
+import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_ELEMENT_ACTIONS, DESKTOP_MODIFIERS, DESKTOP_SYSTEM_KEYS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js";
 import { NOTIFICATIONS_MS, notificationAction, readNotifications, validateNotificationAction, } from "./notifications.js";
 import { waitFor } from "./wait.js";
 const exec = promisify(execFile);
@@ -602,6 +602,11 @@ function validateAction(request) {
         if (Object.keys(request).some((key) => !["op", "key", "modifiers"].includes(key))) {
             throw new Error("A system key accepts only key and modifiers.");
         }
+        if (DESKTOP_SYSTEM_KEYS.includes(request.key)) {
+            if (request.modifiers !== undefined)
+                throw new Error("Media, volume and brightness keys take no modifiers.");
+            return;
+        }
         return validateKey(request);
     }
     if (request.op === "notification-action") {

dist/protocol.d.ts

@@ -261,7 +261,7 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     repeat?: number;
 } | {
     op: "system-key";
-    key: DesktopKey;
+    key: DesktopKey | DesktopSystemKey;
     modifiers?: DesktopModifier[];
 };
 export type DesktopPoint = {
@@ -275,12 +275,15 @@ export type DesktopClick = (typeof DESKTOP_CLICKS)[number];
 export type DesktopDirection = (typeof DESKTOP_DIRECTIONS)[number];
 export type DesktopButton = (typeof DESKTOP_BUTTONS)[number];
 /** Keys that type no character, named; every other key is given as the character it types. */
+/** Media, volume and brightness keys, which the system rather than a window acts on. */
+export declare const DESKTOP_SYSTEM_KEYS: readonly ["play_pause", "next_track", "previous_track", "volume_up", "volume_down", "mute", "brightness_up", "brightness_down", "keyboard_brightness_up", "keyboard_brightness_down"];
 export declare const DESKTOP_KEYS: readonly ["enter", "tab", "escape", "backspace", "delete", "up", "down", "left", "right", "space", "home", "end", "pageup", "pagedown", "help", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20", "keypad0", "keypad1", "keypad2", "keypad3", "keypad4", "keypad5", "keypad6", "keypad7", "keypad8", "keypad9", "keypaddecimal", "keypadmultiply", "keypadplus", "keypadminus", "keypaddivide", "keypadequals", "keypadenter", "keypadclear"];
 export declare const DESKTOP_MODIFIERS: readonly ["command", "control", "option", "shift"];
 export declare const DESKTOP_SELECTIONS: readonly ["text", "cursor_before", "cursor_after"];
 /** Named Accessibility actions; AXPress is desktop_click's route and is not listed separately. */
 export declare const DESKTOP_ELEMENT_ACTIONS: readonly ["AXShowMenu", "AXIncrement", "AXDecrement", "AXConfirm", "AXCancel", "AXPick", "AXRaise", "AXScrollToVisible"];
 export type DesktopNamedKey = (typeof DESKTOP_KEYS)[number];
+export type DesktopSystemKey = (typeof DESKTOP_SYSTEM_KEYS)[number];
 /** A named key, or the one printable character a key types without Shift or Option, such as `,` or `a`. */
 export type DesktopKey = DesktopNamedKey | (string & {});
 export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number];

dist/protocol.js

@@ -18,6 +18,19 @@ export const DESKTOP_CLICKS = ["single", "double", "right", "middle", "triple"];
 export const DESKTOP_DIRECTIONS = ["up", "down", "left", "right"];
 export const DESKTOP_BUTTONS = ["left", "right"];
 /** Keys that type no character, named; every other key is given as the character it types. */
+/** Media, volume and brightness keys, which the system rather than a window acts on. */
+export const DESKTOP_SYSTEM_KEYS = [
+    "play_pause",
+    "next_track",
+    "previous_track",
+    "volume_up",
+    "volume_down",
+    "mute",
+    "brightness_up",
+    "brightness_down",
+    "keyboard_brightness_up",
+    "keyboard_brightness_down",
+];
 export const DESKTOP_KEYS = [
     "enter",
     "tab",

dist/tools.js

@@ -1,4 +1,4 @@
-import { DESKTOP_KEYS, DESKTOP_WAIT_CONDITIONS } from "./protocol.js";
+import { DESKTOP_KEYS, DESKTOP_SYSTEM_KEYS, DESKTOP_WAIT_CONDITIONS } from "./protocol.js";
 import { WAIT_DEFAULT_MS, WAIT_MAX_MS } from "./wait.js";
 // Shared argument shapes; native checks bind each target to the process or window observed.
 const id = { type: "string", minLength: 1, maxLength: 256 };
@@ -95,6 +95,12 @@ const key = {
         },
     ],
 };
+const systemKey = {
+    anyOf: [
+        ...key.anyOf,
+        ...DESKTOP_SYSTEM_KEYS.map((key) => ({ type: "string", const: key })),
+    ],
+};
 const modifiers = {
     type: "array",
     items: {
@@ -823,11 +829,11 @@ Optional modifiers (command, control, option and shift, each at most once) are h
         {
             name: "desktop_system_key",
             op: "system-key",
-            description: `Press and release one system-wide shortcut on ${place}, such as Command-Space for Spotlight, Command-Tab to switch apps, Control-Up for Mission Control, Command-Shift-5 for the screenshot toolbar, or Control-Space to change input source. Use desktop_key for keys meant for a window. This needs no snapshot and has no target: the keys go to whatever is frontmost, as if typed on the keyboard, and they usually move focus, activate apps or open system UI. key and modifiers work as in desktop_key, including refusing characters that need Shift or Option. Any chord with Command and W, Q, H or M is refused, because it would close, quit, hide or minimize whatever is frontmost, or log out; use desktop_close, desktop_quit, desktop_hide or desktop_minimize. The result is unknown even when the keys were delivered, because nothing reads back what they did: observe with desktop_screen, desktop_apps or desktop_windows, close what opened (often desktop_system_key escape), and never blindly repeat the shortcut, since many shortcuts toggle. No keys remain held across calls.`,
+            description: `Press and release one system-wide shortcut on ${place}, such as Command-Space for Spotlight, Command-Tab to switch apps, Control-Up for Mission Control, Command-Shift-5 for the screenshot toolbar, or Control-Space to change input source. Use desktop_key for keys meant for a window. This needs no snapshot and has no target: the keys go to whatever is frontmost, as if typed on the keyboard, and they usually move focus, activate apps or open system UI. key and modifiers work as in desktop_key, including refusing characters that need Shift or Option. Any chord with Command and W, Q, H or M is refused, because it would close, quit, hide or minimize whatever is frontmost, or log out; use desktop_close, desktop_quit, desktop_hide or desktop_minimize. The result is unknown even when the keys were delivered, because nothing reads back what they did: observe with desktop_screen, desktop_apps or desktop_windows, close what opened (often desktop_system_key escape), and never blindly repeat the shortcut, since many shortcuts toggle. No keys remain held across calls. It also presses the media, volume and brightness keys, with no modifiers: play_pause, next_track and previous_track go to whichever app macOS routes media keys to, usually the one playing; volume_up, volume_down and mute change the output volume; brightness_up and brightness_down change the main display's brightness; keyboard_brightness_up and keyboard_brightness_down change the keyboard backlight where there is one. They are sent like the hardware keys, nothing reads back what they changed, and the result is unknown.`,
             parameters: {
                 type: "object",
                 required: ["key"],
-                properties: { key, modifiers },
+                properties: { key: systemKey, modifiers },
             },
         },
         {

docs/tools.md

@@ -109,7 +109,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_element_action` | Perform one Accessibility action an element lists, such as `AXShowMenu`, `AXIncrement`, `AXDecrement`, `AXPick` or `AXCancel` |
 | `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard; where no text control can be observed, type it into the window's keyboard focus |
 | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key; or a sequence of up to 16 keys, or one key up to 32 times |
-| `desktop_system_key` | Press one system-wide shortcut, such as Command-Space or Control-Up, on whatever is frontmost |
+| `desktop_system_key` | Press one system-wide shortcut, such as Command-Space or Control-Up, on whatever is frontmost, or one media, volume or brightness key |
 | `desktop_menu` | Run one menu command by its exact title path |
 | `desktop_dialog_answer` | Press one dialog button by its exact title, replace one dialog text field's value, or choose a path in an Open or Save panel |
 | `desktop_clipboard_write` | Replace the clipboard with text, an image, or file references |
@@ -142,6 +142,8 @@ With `ocr: true`, `desktop_inspect` (in either mode) and `desktop_screen` also r
 
 `desktop_system_key` is for shortcuts that belong to macOS rather than a window: Spotlight (Command-Space), the app switcher (Command-Tab), Mission Control (Control-Up), the screenshot tools (Command-Shift-4 and 5) and the next input source (Control-Space). It takes the same keys and modifiers as `desktop_key`, but no snapshot. The keys are posted as ordinary keyboard input, so they reach whatever is frontmost, and they often move focus, activate an app or open system UI. Nothing reads back what a shortcut did, so the result is `unknown` even after delivery; observe with `desktop_screen`, `desktop_apps` or `desktop_windows`. Any chord with Command and W, Q, H or M is refused before anything is sent, including Command-Shift-Q (log out).
 
+`desktop_system_key` also presses the media, volume and brightness keys: `play_pause`, `next_track`, `previous_track`, `volume_up`, `volume_down`, `mute`, `brightness_up`, `brightness_down`, `keyboard_brightness_up` and `keyboard_brightness_down`. They take no modifiers. These keys have no key code; they are posted as the system-defined events the keyboard sends, so macOS acts on them rather than a window. Media keys go to whichever app macOS routes them to, usually the one playing, and the brightness keys change the main display and the keyboard backlight where macOS supports it. The result is `unknown` after delivery, because nothing reads back the volume, brightness or playback state. Don't blindly repeat `play_pause` or `mute`; they toggle.
+
 `desktop_notifications` inspects Notification Center's windows on screen: its banners and alerts, and its panel when open (click the clock with `desktop_menubar_click` to open it). Each notification has an `id`, its app, title, subtitle, body and time where they can be read, its raw `texts` and `description`, and its `actions`: `open`, plus the custom actions Notification Center lists for it by exact name, such as Close, Show, Options or the posting app's own. `desktop_notification_action` takes that `id` and one of those actions. `open` is the notification's default click (on a stack, it expands the group); `close` runs Close; any other name runs that one listed custom action through Accessibility. Only Notification Center's elements list or run custom actions, at most 8 per element, each one line of at most 64 characters, and a name listed twice is never offered. A notification that has gone or changed is refused before input. Afterwards the notifications are read again: the result is `completed` only when the notification is gone, and `unknown` otherwise. System alerts such as permission prompts are dialogs, so `desktop_notifications` lists them under `system_alerts` for `desktop_dialog` and `desktop_dialog_answer` and never acts on them. This is built against the Notification Center structure of macOS 11 to 15.
 
 An action refused because its window was minimized after inspection reports `WINDOW_MINIMIZED`, with the low-level reason as its `cause`. Restore the window with `desktop_restore`, then inspect it again.

native/patches/README.md

@@ -415,6 +415,14 @@ also refuses it before dispatch unless the target process is Notification Center
 exactly that name. It performs the matching descriptor and reports the requested name, which the Bridge
 receipt requires. No Bridge operation is added. Built against the macOS 11 to 15 structure; tests use fixtures.
 
+`peekaboo-system-defined-key.patch` adds `systemDefinedKey`, which presses one media, volume or brightness
+key. These keys have no key code: the keyboard sends them as `NX_SYSDEFINED` events with the
+`NX_SUBTYPE_AUX_CONTROL_BUTTONS` subtype, which upstream cannot post. The host builds the press and the
+release before posting either, then posts both to the HID tap, where the system rather than a window acts on
+them. Like `moveMouse`, it needs only event posting, has a global target, and reports one foreground
+dispatch whose effect is not verified. The keys are play/pause, next, previous, volume up and down, mute,
+display brightness up and down, and keyboard backlight up and down.
+
 `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision,
 and assembles the ordered patch stack in a private Git index. A build compares the checkout
 with each complete ordered prefix, since later patches can change earlier patch contexts.