feat: type insertions where no text control can be observed

dist/client.js

@@ -640,6 +640,9 @@ function validateAction(request) {
         if (typeof request.text !== "string" || !request.text || request.text.length > 8192) {
             throw new Error("Inserted text must contain 1 to 8192 UTF-16 code units.");
         }
+        if (request.route !== undefined && request.route !== "paste" && request.route !== "type") {
+            throw new Error("Choose the paste or type route, or omit route.");
+        }
         return;
     }
     if (typeof request.element !== "string" || !request.element || request.element.length > 256) {

dist/protocol.d.ts

@@ -182,6 +182,7 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     op: "insert";
     snapshot: string;
     text: string;
+    route?: "paste" | "type";
 } | {
     op: "element-action";
     snapshot: string;

dist/tools.js

@@ -689,13 +689,14 @@ Optional modifiers (command, control, option and shift, each at most once) are h
         {
             name: "desktop_insert",
             op: "insert",
-            description: `Insert literal text at the current caret or replace the current selection in the control focused in a fresh desktop_inspect result, preserving surrounding text. The GUI sends the entire Unicode/multiline string through one temporary clipboard paste bound to the exact process, window, and focused control; it never turns newlines into Enter keys or falls back to typing. Requires readable text/selection and allowed macOS clipboard reading to preserve prior contents. The caret/selection can change after inspection. Clipboard restoration requires a meaningful observed edit; after uncertain delivery, the replacement may remain on the clipboard and newer copied contents are preserved. Read consumption, clipboard_cleanup, and clipboard_ownership separately from delivery. Reserved ownership refuses later automated clipboard writes until the edit is verified or the exact receiving process exits; restarting ${app} does not silently clear it. An exact receiver in the active frontmost app uses the targeted paste chord directly. Other targets require a guarded blank native title-bar click that preserves the editor and selection; unsupported window chrome is refused. The chosen route never changes after input begins and does not bring the app to the front. Original and resulting field values must fit 65,536 UTF-16 units. Pass snapshot_id as snapshot. The snapshot is single-use and input is never automatically replayed. Inspect after uncertain delivery before another action.`,
+            description: `Insert literal text at the current caret or replace the current selection in the window bound by a fresh desktop_inspect result, preserving surrounding text. When the snapshot reports a focused control, the GUI sends the entire Unicode/multiline string through one temporary clipboard paste bound to the exact process, window, and focused control (the paste route); it never turns newlines into Enter keys. Requires readable text/selection and allowed macOS clipboard reading to preserve prior contents. The caret/selection can change after inspection. Clipboard restoration requires a meaningful observed edit; after uncertain delivery, the replacement may remain on the clipboard and newer copied contents are preserved. Read consumption, clipboard_cleanup, and clipboard_ownership separately from delivery. Reserved ownership refuses later automated clipboard writes until the edit is verified or the exact receiving process exits; restarting ${app} does not silently clear it. An exact receiver in the active frontmost app uses the targeted paste chord directly. Other targets require a guarded blank native title-bar click that preserves the editor and selection; unsupported window chrome is refused. The chosen route never changes after input begins and does not bring the app to the front. Original and resulting field values must fit 65,536 UTF-16 units. When the snapshot reports no focused control, as in terminals, canvases, custom views and web editors that expose no Accessibility text, the text is typed instead (the type route) into whatever has keyboard focus in that window (receiver window_focus); before each character the app's focused element must still be inside the window and the window must be the app's key window, or typing stops. Pass route type to type into an observed control whose text can't be read, such as a terminal, or route paste to refuse typing. Typing leaves the clipboard untouched, takes at most 1,000 UTF-16 units, refuses newlines, tabs and other control characters (press those with desktop_key), and is refused while secure keyboard entry is on. Characters go to the window in the background, and nothing reads back what the receiver did, so a typed insertion is unknown unless the host confirms an observed control's new value; inspect the window before acting again. Pass snapshot_id as snapshot. The snapshot is single-use and input is never automatically replayed. Inspect after uncertain delivery before another action.`,
             parameters: {
                 type: "object",
                 required: ["snapshot", "text"],
                 properties: {
                     snapshot: id,
                     text: { type: "string", minLength: 1, maxLength: 8192 },
+                    route: { type: "string", enum: ["paste", "type"] },
                 },
             },
         },

docs/tools.md

@@ -74,7 +74,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_select` | Select literal text in a field, or put the caret before or after it |
 | `desktop_set_value` | Set a slider, stepper or level indicator to a number, or a checkbox, switch or disclosure triangle on or off, and read it back |
 | `desktop_element_action` | Perform one Accessibility action an element lists, such as `AXShowMenu`, `AXIncrement`, `AXDecrement`, `AXPick` or `AXCancel` |
-| `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard |
+| `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard; where no text control can be observed, type it into the window's keyboard focus |
 | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key |
 | `desktop_system_key` | Press one system-wide shortcut, such as Command-Space or Control-Up, on whatever is frontmost |
 | `desktop_menu` | Run one menu command by its exact title path |
@@ -95,6 +95,8 @@ With `ocr: true`, `desktop_inspect` (in either mode) and `desktop_screen` also r
 
 `desktop_dialog_answer` with `path` chooses an absolute path in an Open or Save panel without pressing its default button. An existing folder becomes the panel's folder. Any other path names a file in an existing folder: a Save panel takes its folder and file name, and an Open panel selects it. Unlike buttons and fields, this is foreground input: the host activates the panel's app, checks that the exact panel has keyboard focus before each key, and uses the panel's path field or Go to Folder. The result is `completed` only when the panel reads back the folder and, in a Save panel, the name. Choosing a file in an Open panel, or a panel that shows no folder, is `unknown`; list the dialog before pressing its default button.
 
+`desktop_insert` pastes into the focused control the snapshot reports, and restores the clipboard once it reads the edit back. When the snapshot reports no focused control, as in terminals, canvases, custom views and many web editors, it types the text into whatever has keyboard focus in that window instead, with the same check `desktop_key` makes before each character: the app's focused element must still be inside the window, and the window must be the app's key window. Pass `route: "type"` to type into an observed control whose text can't be read, such as a terminal's text area, or `route: "paste"` to refuse typing. Typing doesn't touch the clipboard, takes at most 1,000 UTF-16 code units, and refuses newlines, tabs and other control characters, because typing them presses Return or Tab; press those with `desktop_key` where you mean to. It is refused while secure keyboard entry is on, as for a password field or Terminal's Secure Keyboard Entry. A typed insertion is `unknown` unless the host reads back an observed control's new value, so inspect the window before acting again.
+
 `desktop_inspect` lists the actions each element offers in `attributes.actions`. A pop-up button has no settable value: click it, then pick the item from the next observation, which puts an open menu's items first.
 
 `desktop_system_key` is for shortcuts that belong to macOS rather than a window: Spotlight (Command-Space), the app switcher (Command-Tab), Mission Control (Control-Up), the screenshot tools (Command-Shift-4 and 5) and the next input source (Control-Space). It takes the same keys and modifiers as `desktop_key`, but no snapshot. The keys are posted as ordinary keyboard input, so they reach whatever is frontmost, and they often move focus, activate an app or open system UI. Nothing reads back what a shortcut did, so the result is `unknown` even after delivery; observe with `desktop_screen`, `desktop_apps` or `desktop_windows`. Any chord with Command and W, Q, H or M is refused before anything is sent, including Command-Shift-Q (log out).
@@ -103,7 +105,7 @@ An action refused because its window was minimized after inspection reports `WIN
 
 ## Not supported
 
-- **Insertion outside text fields.** `desktop_insert` needs an observed focused text control with a readable selection.
+- **Confirmed insertion outside text fields.** Where `desktop_insert` can't observe a text control with a readable selection, such as terminals, canvases and custom views, it types the text instead, and the result is `unknown`, because nothing can read back what the receiver did with it. Typed insertion can't contain newlines or tabs.
 - **Confirmed drops between windows.** A drag into another window or app, or any drag with modifier keys, is always reported as `unknown`, because nothing can confirm the drop. A path can't cross into another window: a drag between windows is one straight line.
 - **Moving a window to another Space.** Since macOS 14.5 only the Dock can move windows between Spaces, so there is no tool for it. To do it by hand, open Mission Control with `desktop_system_key` (`control`, `up`), inspect it, and drag the window onto the other Space with a foreground `desktop_drag`. The result is `unknown`; check with `desktop_spaces`.
 - **Automatic retries.** By design, nothing is retried. After a `refused` or `unknown` result, observe again before deciding whether to act.

native/sources/client/actions.swift

@@ -1,9 +1,13 @@
+import Carbon
 import CoreGraphics
 import Foundation
 import PeekabooAutomationKit
 import PeekabooBridge
 import PeekabooFoundation
 
+/// Typed insertion checks focus before every character, so it is kept short enough to finish well within a request.
+private let maximumTypedInsertion = 1000
+
 private struct ActionRequest: Decodable {
 	enum Operation: String, Decodable {
 		case click, type, key, insert, select, scroll, drag, hover, press
@@ -32,6 +36,7 @@ private struct ActionRequest: Decodable {
 	let value: UIElementValue?
 	let to_snapshot: String?
 	let path: [PointerPoint]?
+	var route: String?
 
 	/// Modifier keys and drags between windows need real global input; other drags stay in the background.
 	var dragsInForeground: Bool {
@@ -66,6 +71,9 @@ private struct ActionRequest: Decodable {
 		if op != .drag, op != .press, button != nil || duration_ms != nil {
 			throw ActionError("Only dragging or pressing accepts a button or duration.")
 		}
+		if op != .insert, route != nil {
+			throw ActionError("Only insertion accepts a route.")
+		}
 		if op != .click, op != .scroll, op != .hover, op != .press, point != nil {
 			throw ActionError("Only clicks, scrolling, hovering, or pressing accept a target point.")
 		}
@@ -171,6 +179,21 @@ private struct ActionRequest: Decodable {
 			guard element == nil, key == nil, let text, !text.isEmpty, text.utf16.count <= 8192 else {
 				throw ActionError("Insert nonempty text of at most 8,192 UTF-16 code units into the observed focused control.")
 			}
+			guard route == nil || route == "paste" || route == "type" else {
+				throw ActionError("Choose the paste or type route, or omit route.")
+			}
+			if route == "type" {
+				guard text.utf16.count <= maximumTypedInsertion else {
+					throw ActionError("Typed insertion takes at most \(maximumTypedInsertion) UTF-16 code units; insert longer text in parts.")
+				}
+				// Typing a newline or tab presses Return or Tab, which can run a command or move focus.
+				guard !text.unicodeScalars.contains(where: { $0.properties.generalCategory == .control }) else {
+					throw ActionError(
+						"Typed insertion can't contain newlines, tabs or other control characters, because typing them presses keys such as Return.",
+						hint: "Insert each line separately and press desktop_key enter or tab between them only where you mean to."
+					)
+				}
+			}
 		case .key:
 			guard element == nil, text == nil, let key, serviceKey(key) != nil else {
 				throw ActionError("Choose a named key or the one printable character a key types, such as , or a.")
@@ -359,10 +382,27 @@ func nativeAction(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHands
 			}
 			try checkElementRequest(request, observed)
 		}
-		if request.op == .insert, context.focusedElement == nil {
-			throw ActionError("The snapshot has no exact focused control. Click the text field, then inspect the window again.")
+		// Without an observed focused control there's no text to read back, so insertion types into window focus.
+		let typesInsertion = request.op == .insert
+			&& (request.route == "type" || (request.route == nil && context.focusedElement == nil))
+		if request.op == .insert, request.route == "paste", context.focusedElement == nil {
+			throw ActionError("The snapshot has no exact focused control to paste into. Click the text field and inspect the window again, or omit route to type into the window's focus.")
+		}
+		if typesInsertion {
+			var effective = request
+			effective.route = "type"
+			try effective.validate()
+			guard (handshake.enabledOperations ?? handshake.supportedOperations).contains(.exactWindowTargetedTypeActions) else {
+				throw ActionError("The native runtime does not allow typed insertion. Update the desktop runtime before acting.")
+			}
+			if IsSecureEventInputEnabled() {
+				throw ActionError(
+					"Secure keyboard entry is on, as for a password field or a terminal's Secure Keyboard Entry, so typed text can't be delivered. Nothing was typed.",
+					hint: "Use a field that isn't secure, or turn off Secure Keyboard Entry in the app."
+				)
+			}
 		}
-		if request.op == .insert {
+		if request.op == .insert, !typesInsertion {
 			// The GUI owns the snapshot lease and clipboard transaction through consumption and cleanup.
 			invoked = true
 			let insertion = try await client.literalInsert(snapshot: request.snapshot, text: request.text!)
@@ -502,7 +542,20 @@ func nativeAction(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHands
 				))
 			}
 		case .insert:
-			throw ActionError("Literal insertion must use its GUI-owned transaction.")
+			// Typing leaves the clipboard alone. The host checks focus before each character, as for keys.
+			if let focusedElement = context.focusedElement {
+				result.receiver = "observed_control"
+				evidence = try await ActionEvidence(client.typeActionsWithOutcome(
+					[.text(request.text!)], cadence: .fixed(milliseconds: 2), snapshotId: request.snapshot,
+					target: .init(windowIdentity: identity, windowBounds: bounds, focusedElement: focusedElement)
+				))
+			} else {
+				result.receiver = "window_focus"
+				evidence = try await ActionEvidence(client.typeActionsWithOutcome(
+					[.text(request.text!)], cadence: .fixed(milliseconds: 2), snapshotId: request.snapshot,
+					expectedWindowIdentity: identity, expectedWindowBounds: bounds
+				))
+			}
 		case .select:
 			let selected = try await client.selectText(
 				target: request.element!,
@@ -535,6 +588,15 @@ func nativeAction(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHands
 		default:
 			result.outcome = outcome.evidence == .operationStillRunning || !verified ? "unknown" : "completed"
 		}
+		// Delivered keystrokes don't show what the receiver did with them unless the host read the change back.
+		if request.op == .insert, result.outcome == "completed", outcome.state != .confirmedChange {
+			result.outcome = "unknown"
+			result.error = ActionMessage(
+				code: "INSERTION_UNVERIFIED",
+				message: "The text was typed into the window's keyboard focus, but nothing could read back what the receiver did with it.",
+				hint: "Inspect the window, or capture it with desktop_screen, to check the text before acting again; do not repeat the insertion blindly."
+			)
+		}
 		if !verified, result.outcome != "refused" {
 			result.outcome = "unknown"
 			result.error = ActionMessage(

native/sources/runtime/runtime.swift

@@ -84,7 +84,7 @@ private final class Desktop {
 						.listApplications, .listWindows, .listMenus, .menuCommand, .desktopObservation,
 						.createSnapshot, .cleanSnapshot, .ownsSnapshot, .getDetectionResult, .beginSnapshotMutation,
 						.finishSnapshotMutation,
-						.targetedClick, .exactWindowTargetedClick, .setValue, .performAction, .exactWindowTargetedHotkey, .hotkey,
+						.targetedClick, .exactWindowTargetedClick, .setValue, .performAction, .exactWindowTargetedHotkey, .hotkey, .exactWindowTargetedTypeActions,
 						.selectText, .literalInsert, .clipboardTextRead, .clipboardImageRead, .clipboardFilesRead, .clipboardTextWrite, .clipboardImageWrite, .clipboardFilesWrite, .targetedScroll, .exactWindowDrag, .drag, .moveMouse,
 						.createExactWindowHeldPointerOwner, .beginExactWindowHeldPointer, .releaseExactWindowHeldPointer,
 						.revokeExactWindowHeldPointer, .disconnectExactWindowHeldPointerOwner,

src/client.ts

@@ -768,6 +768,9 @@ function validateAction(request: DesktopAction) {
 		if (typeof request.text !== "string" || !request.text || request.text.length > 8192) {
 			throw new Error("Inserted text must contain 1 to 8192 UTF-16 code units.");
 		}
+		if (request.route !== undefined && request.route !== "paste" && request.route !== "type") {
+			throw new Error("Choose the paste or type route, or omit route.");
+		}
 		return;
 	}
 	if (typeof request.element !== "string" || !request.element || request.element.length > 256) {

src/protocol.ts

@@ -100,7 +100,7 @@ export type DesktopRequest =
 		duration_ms?: number;
 	}
 	| { op: "type"; snapshot: string; element: string; text: string }
-	| { op: "insert"; snapshot: string; text: string }
+	| { op: "insert"; snapshot: string; text: string; route?: "paste" | "type" }
 	| { op: "element-action"; snapshot: string; element: string; action: DesktopElementAction }
 	| { op: "set-value"; snapshot: string; element: string; value: number | boolean }
 	| {