feat(macos): draw a ghost cursor for native app backend actions

apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift

@@ -84,7 +84,8 @@ func requireSessionApplication(bundleId: String?) throws -> NSRunningApplication
 
 func pressInBackground(
   _ request: MouseClickRequest,
-  app: NSRunningApplication
+  app: NSRunningApplication,
+  cursor: GhostCursor?
 ) throws -> BackgroundPressResponse {
   guard !request.doubleClick, request.holdMs <= 0 else {
     throw refusal(
@@ -94,6 +95,7 @@ func pressInBackground(
     )
   }
   let point = CGPoint(x: request.x, y: request.y)
+  cursor?.move(to: point)
   guard let target = resolvePressTarget(app: app, point: point),
     let mechanism = perform(target)
   else {
@@ -105,6 +107,7 @@ func pressInBackground(
     guard perform(target) != nil else { break }
     clicks += 1
   }
+  cursor?.pulse()
   return BackgroundPressResponse(
     x: request.x,
     y: request.y,
@@ -121,14 +124,19 @@ func pressInBackground(
 func typeInBackground(
   text: String,
   delayMs: Int,
-  app: NSRunningApplication
+  app: NSRunningApplication,
+  cursor: GhostCursor?
 ) throws -> BackgroundTextResponse {
   let appElement = AXUIElementCreateApplication(app.processIdentifier)
   let focused = elementAttribute(appElement, attribute: kAXFocusedUIElementAttribute as String)
+  if let focused, let rect = rectAttribute(focused) {
+    cursor?.move(to: CGPoint(x: rect.x + rect.width / 2, y: rect.y + rect.height / 2))
+  }
   let focusedRole = focused.map(role(of:))
   if text == "\n", let focused, actionNames(of: focused).contains(kAXConfirmAction as String),
     AXUIElementPerformAction(focused, kAXConfirmAction as CFString) == .success
   {
+    cursor?.pulse()
     return BackgroundTextResponse(
       bundleId: app.bundleIdentifier, mechanism: .axConfirm, role: focusedRole,
       windowTitle: windowTitle(of: focused))
@@ -138,11 +146,13 @@ func typeInBackground(
     AXUIElementSetAttributeValue(focused, kAXSelectedTextAttribute as CFString, text as CFString)
       == .success
   {
+    cursor?.pulse()
     return BackgroundTextResponse(
       bundleId: app.bundleIdentifier, mechanism: .axSelectedText, role: focusedRole,
       windowTitle: windowTitle(of: focused))
   }
   try postText(text, delayMs: delayMs, pid: app.processIdentifier)
+  cursor?.pulse()
   return BackgroundTextResponse(
     bundleId: app.bundleIdentifier, mechanism: .keyEvents, role: focusedRole,
     windowTitle: focused.flatMap(windowTitle(of:)))
@@ -152,8 +162,10 @@ func typeInBackground(
 func fillInBackground(
   point: CGPoint,
   text: String,
-  app: NSRunningApplication
+  app: NSRunningApplication,
+  cursor: GhostCursor?
 ) throws -> BackgroundTextResponse {
+  cursor?.move(to: point)
   let hit = elementAtPoint(in: app, point: point)
   let chain = hit.map(pressSearchChain) ?? []
   let fallback = actionWindow(app: app, hit: hit).flatMap { window in
@@ -169,6 +181,7 @@ func fillInBackground(
   else {
     throw refusal(.noTextInput, "the text input refused the new value", app: app)
   }
+  cursor?.pulse()
   return BackgroundTextResponse(
     bundleId: app.bundleIdentifier, mechanism: .axValue, role: role(of: input),
     windowTitle: windowTitle(of: input))
@@ -179,7 +192,8 @@ func scrollInBackground(
   direction: String,
   amount: Double?,
   pixels: Double?,
-  app: NSRunningApplication
+  app: NSRunningApplication,
+  cursor: GhostCursor?
 ) throws -> BackgroundScrollResponse {
   let isVertical = direction == "up" || direction == "down"
   guard isVertical || direction == "left" || direction == "right" else {
@@ -197,6 +211,7 @@ func scrollInBackground(
     pixels: pixels
   )
   let center = CGPoint(x: frame.midX, y: frame.midY)
+  cursor?.move(to: center)
   let revealsLaterContent = direction == "down" || direction == "right"
   guard
     performScrollBarScroll(
@@ -208,6 +223,7 @@ func scrollInBackground(
   else {
     throw refusal(.noScrollBar, "no scroll area with a settable scroll bar under the window center", app: app)
   }
+  cursor?.pulse()
   // Reported as the equivalent drag, start to end, the way the runner reports a desktop scroll.
   let half = travel / 2
   let sign: Double = revealsLaterContent ? 1 : -1

apple/macos-helper/Sources/AgentDeviceMacOSHelper/GhostCursor.swift

@@ -0,0 +1,141 @@
+import AppKit
+import QuartzCore
+
+/// A drawn pointer that shows where a background action lands while the user keeps the real
+/// one. It appears just above and left of the target, glides onto it, and pulses when the action
+/// is delivered: about 0.3 s per action. It holds no state between helper processes.
+final class GhostCursor {
+  private static let size = CGSize(width: 64, height: 64)
+  /// Where the arrow's tip sits inside the panel, from its top-left corner; it leaves room for
+  /// the pulse ring's full radius.
+  private static let tipOffset = CGPoint(x: 22, y: 22)
+  /// Where the glide starts, relative to the target.
+  private static let approach = CGVector(dx: -36, dy: -36)
+
+  private let panel: NSPanel
+  private let view: GhostCursorView
+  private let primaryScreenHeight: CGFloat
+
+  /// Nil when no display is attached; the action still runs, just unseen.
+  static func show() -> GhostCursor? {
+    _ = NSApplication.shared
+    NSApp.setActivationPolicy(.accessory)
+    guard let primary = NSScreen.screens.first else { return nil }
+    return GhostCursor(primaryScreenHeight: primary.frame.height)
+  }
+
+  private init(primaryScreenHeight: CGFloat) {
+    self.primaryScreenHeight = primaryScreenHeight
+    view = GhostCursorView(frame: CGRect(origin: .zero, size: Self.size), tip: Self.tipOffset)
+    panel = NSPanel(
+      contentRect: CGRect(origin: .zero, size: Self.size),
+      styleMask: [.borderless, .nonactivatingPanel],
+      backing: .buffered,
+      defer: false
+    )
+    panel.isOpaque = false
+    panel.backgroundColor = .clear
+    panel.hasShadow = false
+    panel.ignoresMouseEvents = true
+    panel.level = .screenSaver
+    panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary]
+    panel.contentView = view
+  }
+
+  /// Glides onto a point in global top-left coordinates, the space AX and CGEvent share.
+  func move(to target: CGPoint, duration: TimeInterval = 0.15) {
+    let start = CGPoint(x: target.x + Self.approach.dx, y: target.y + Self.approach.dy)
+    place(at: start)
+    panel.orderFrontRegardless()
+    let steps = max(1, Int(duration / 0.016))
+    for step in 1...steps {
+      let t = Double(step) / Double(steps)
+      let eased = 1 - pow(1 - t, 3)
+      place(at: CGPoint(x: start.x + (target.x - start.x) * eased, y: start.y + (target.y - start.y) * eased))
+      flush(for: 0.016)
+    }
+  }
+
+  /// A ring at the tip, marking the moment the action was delivered; nothing when the cursor never
+  /// moved onto a target.
+  func pulse(duration: TimeInterval = 0.12) {
+    guard panel.isVisible else { return }
+    let steps = max(1, Int(duration / 0.016))
+    for step in 1...steps {
+      view.ringProgress = CGFloat(step) / CGFloat(steps)
+      view.display()
+      flush(for: 0.016)
+    }
+  }
+
+  func hide() {
+    panel.orderOut(nil)
+    CATransaction.flush()
+  }
+
+  private func place(at tip: CGPoint) {
+    panel.setFrameOrigin(
+      CGPoint(
+        x: tip.x - Self.tipOffset.x,
+        y: primaryScreenHeight - tip.y - Self.size.height + Self.tipOffset.y
+      )
+    )
+  }
+
+  private func flush(for interval: TimeInterval) {
+    CATransaction.flush()
+    RunLoop.current.run(until: Date(timeIntervalSinceNow: interval))
+  }
+}
+
+private final class GhostCursorView: NSView {
+  private let tip: CGPoint
+  var ringProgress: CGFloat?
+
+  init(frame: CGRect, tip: CGPoint) {
+    self.tip = tip
+    super.init(frame: frame)
+  }
+
+  required init?(coder: NSCoder) {
+    nil
+  }
+
+  override var isFlipped: Bool { true }
+
+  override func draw(_ dirtyRect: NSRect) {
+    NSColor.clear.setFill()
+    dirtyRect.fill()
+    if let progress = ringProgress {
+      let radius = 6 + 14 * progress
+      let ring = NSBezierPath(
+        ovalIn: CGRect(x: tip.x - radius, y: tip.y - radius, width: radius * 2, height: radius * 2)
+      )
+      ring.lineWidth = 3
+      NSColor.systemPurple.withAlphaComponent(0.85 * (1 - progress)).setStroke()
+      ring.stroke()
+    }
+    let arrow = NSBezierPath()
+    arrow.move(to: tip)
+    arrow.line(to: CGPoint(x: tip.x, y: tip.y + 22))
+    arrow.line(to: CGPoint(x: tip.x + 6, y: tip.y + 17))
+    arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 26))
+    arrow.line(to: CGPoint(x: tip.x + 14, y: tip.y + 24))
+    arrow.line(to: CGPoint(x: tip.x + 10, y: tip.y + 15))
+    arrow.line(to: CGPoint(x: tip.x + 17, y: tip.y + 15))
+    arrow.close()
+    arrow.lineJoinStyle = .round
+    let shadow = NSShadow()
+    shadow.shadowBlurRadius = 3
+    shadow.shadowOffset = CGSize(width: 0, height: -1)
+    shadow.shadowColor = NSColor.black.withAlphaComponent(0.35)
+    NSGraphicsContext.saveGraphicsState()
+    shadow.set()
+    NSColor.systemPurple.setFill()
+    arrow.fill()
+    NSGraphicsContext.restoreGraphicsState()
+    arrow.lineWidth = 1.5
+    NSColor.white.setStroke()
+    arrow.stroke()
+  }
+}

apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift

@@ -419,7 +419,9 @@ struct AgentDeviceMacOSHelper {
     if surface == "app" {
       let app = try requireSessionApplication(bundleId: bundleId)
       return SuccessEnvelope(
-        data: try pressInBackground(request, app: app)
+        data: try withGhostCursor(arguments) { cursor in
+          try pressInBackground(request, app: app, cursor: cursor)
+        }
       )
     }
     try pressAtPosition(request)
@@ -450,7 +452,9 @@ struct AgentDeviceMacOSHelper {
       bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
     )
     return SuccessEnvelope(
-      data: try typeInBackground(text: text, delayMs: delayMs, app: app)
+      data: try withGhostCursor(arguments) { cursor in
+        try typeInBackground(text: text, delayMs: delayMs, app: app, cursor: cursor)
+      }
     )
   }
 
@@ -467,7 +471,10 @@ struct AgentDeviceMacOSHelper {
       bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
     )
     return SuccessEnvelope(
-      data: try fillInBackground(point: CGPoint(x: x, y: y), text: text, app: app)
+      data: try withGhostCursor(arguments) { cursor in
+        try fillInBackground(
+          point: CGPoint(x: x, y: y), text: text, app: app, cursor: cursor)
+      }
     )
   }
 
@@ -484,12 +491,15 @@ struct AgentDeviceMacOSHelper {
       bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId)
     )
     return SuccessEnvelope(
-      data: try scrollInBackground(
-        direction: direction,
-        amount: amount,
-        pixels: pixels,
-        app: app
-      )
+      data: try withGhostCursor(arguments) { cursor in
+        try scrollInBackground(
+          direction: direction,
+          amount: amount,
+          pixels: pixels,
+          app: app,
+          cursor: cursor
+        )
+      }
     )
   }
 
@@ -542,6 +552,13 @@ private func optionValue(arguments: [String], name: String) -> String? {
   return arguments[index + 1]
 }
 
+/// Runs a background action under the ghost cursor when the host asked for one.
+private func withGhostCursor<T>(_ arguments: [String], _ body: (GhostCursor?) throws -> T) throws -> T {
+  let cursor = arguments.contains("--ghost-cursor") ? GhostCursor.show() : nil
+  defer { cursor?.hide() }
+  return try body(cursor)
+}
+
 private func positiveDoubleOption(arguments: [String], name: String) throws -> Double? {
   guard let raw = optionValue(arguments: arguments, name: name) else { return nil }
   guard let value = Double(raw), value.isFinite, value > 0 else {

docs/adr/0031-macos-native-app-backend.md

@@ -67,6 +67,11 @@ point. Sending an element identity instead would be a new dispatch path with its
 row, and an `AXUIElement` cannot outlive the one-shot helper process that resolved it, so an
 identity would be a tree path re-resolved against a tree that may have changed.
 
+**Ghost cursor.** Each helper action draws its own pointer, glides onto the target from a short
+fixed offset, and pulses on delivery, so a person can follow the agent without losing the real
+pointer. It costs about 0.3 s per action and keeps no state between helper processes; a persistent
+pointer would need a long-lived helper. `AGENT_DEVICE_MACOS_GHOST_CURSOR=0` disables it.
+
 ## Rejected alternatives
 
 - **Suppressing Automation Mode.** `automationmodetool` removes the authentication prompt, not the

docs/adr/README.md

@@ -32,7 +32,7 @@
 | [0028 Capability-Family Cell Vocabulary — One Runtime Source (Proposed)](0028-capability-family-cell-vocabulary.md) | adding a capability operation family, `UnavailablePlatformRuntimeFacts` / `UNAVAILABLE_CELLS`, `INTERACTOR_OPERATIONS`, and why an eight-package fan-out recurs |
 | [0029 Daemon Policy](0029-daemon-policy.md) | `AGENT_DEVICE_DAEMON_POLICY`, confining a daemon's commands, devices, or device shutdown, and where operator rules are enforced for batch/replay steps |
 | [0030 Process Lock Exclusion](0030-process-lock-exclusion.md) | process-lock publication/reclaim/release, retained mutation guards, and the single-protocol upgrade boundary |
-| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, and why pointer actions are accessibility actions only |
+| [0031 macOS Native App Backend](0031-macos-native-app-backend.md) | `AGENT_DEVICE_MACOS_APP_BACKEND`, driving macOS app sessions without XCTest Automation Mode, why pointer actions are accessibility actions only, and the ghost cursor |
 
 ADRs record *why*; the registries and gates they describe are the living source of truth — when
 prose and a registry disagree, the registry wins and the ADR needs a follow-up.