fix(agent): preserve recorded cursor visibility

electron/ai-edition/agent-tools.test.ts

@@ -1686,6 +1686,29 @@ describe("getCursorTrack", () => {
 		expect(result.resultJson).not.toContain('virtualSec":');
 	});
 
+	it("reports a hidden click and the return to visibility through the tool payload", () => {
+		const samples = [
+			{ timeMs: 0, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },
+			{ timeMs: 50, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" },
+			{ timeMs: 100, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },
+			{ timeMs: 1000, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" },
+		];
+		const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", {
+			cursorTelemetry: { load: { status: "ok", assetId: "asset_1", samples } },
+		});
+		const payload = JSON.parse(result.resultJson);
+
+		expect(result.ok).toBe(true);
+		expect(payload.points).toContainEqual({
+			atSec: 0.05,
+			cx: 0.5,
+			cy: 0.5,
+			kind: "click",
+			visible: false,
+		});
+		expect(payload.points).toContainEqual({ atSec: 0.1, cx: 0.5, cy: 0.5 });
+	});
+
 	it("says 'no-sidecar' when the asset was checked and has none", () => {
 		const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", {
 			cursorTelemetry: { load: { status: "no-sidecar", assetId: "asset_1" } },

electron/ai-edition/deep-agent/service.ts

@@ -154,7 +154,7 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
 	getTranscript:
 		"Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.",
 	getCursorTrack:
-		"Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.",
+		"Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`; points where the pointer was hidden carry `visible:false`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.",
 	getTranscriptWords:
 		'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.',
 	setWordText:

src/lib/ai-edition/timeline/cursor-track.test.ts

@@ -76,6 +76,43 @@ describe("buildCursorTrack", () => {
 		expect(click?.atSec).toBeCloseTo(13 * 0.05, 2);
 	});
 
+	it("keeps both visibility changes even when a stationary cursor is sampled below their rate", () => {
+		const samples = Array.from({ length: 8 }, (_, i) => ({
+			timeMs: i * 50,
+			cx: 0.5,
+			cy: 0.5,
+			assetId: "arrow",
+			interactionType: "move",
+			visible: i === 3 ? false : undefined,
+		}));
+		const track = build(samples, 0.1);
+
+		expect(track.points.find((point) => point.atSec === 0.15)).toMatchObject({ visible: false });
+		expect(track.points.find((point) => point.atSec === 0.2)).toBeDefined();
+		expect(track.points.find((point) => point.atSec === 0.2)).not.toHaveProperty("visible");
+	});
+
+	it("marks a recorded click as hidden instead of presenting it as visible to the agent", () => {
+		const samples = [
+			{ timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move", visible: true },
+			{ timeMs: 500, cx: 0.5, cy: 0.5, interactionType: "click", visible: false },
+			{ timeMs: 1000, cx: 0.5, cy: 0.5, interactionType: "move", visible: true },
+		];
+		expect(build(samples).points.find((point) => point.kind === "click")).toMatchObject({
+			atSec: 0.5,
+			visible: false,
+		});
+	});
+
+	it("treats legacy samples without visibility as visible, omitting the default field", () => {
+		const legacy = sweep(40);
+		const explicit = legacy.map((sample) => ({ ...sample, visible: true }));
+		const legacyTrack = build(legacy);
+
+		expect(build(explicit).points).toEqual(legacyTrack.points);
+		expect(legacyTrack.points.every((point) => !("visible" in point))).toBe(true);
+	});
+
 	it("drops to a coarser rate rather than blowing the ceiling, and says so", () => {
 		// 40 minutes at 20 Hz: 5 Hz would be 12 000 points.
 		const track = buildCursorTrack({

src/lib/ai-edition/timeline/cursor-track.ts

@@ -1,5 +1,5 @@
 // The recorded cursor track the agent reads: where the pointer was, when, and
-// what shape it had. Pure — no fs, no IPC. The caller supplies the samples.
+// what shape it had, and when it was visible. Pure — no fs, no IPC. The caller supplies the samples.
 //
 // ponytail: this is an OBSERVATION, not an interpretation, and the distinction
 // is the whole point of the module. Its predecessor handed the model a list of
@@ -28,6 +28,8 @@ export interface CursorTrackSample {
 	timeMs: number;
 	cx: number;
 	cy: number;
+	/** Absent in older sidecars, where the pointer is treated as visible. */
+	visible?: boolean;
 	/** The cursor BITMAP's id, not a media asset: the sidecar stores one entry per
 	 *  distinct pointer image (arrow, hand, text caret, resize…). A change means the
 	 *  pointer shape changed, which is why these points are never dropped. */
@@ -44,6 +46,8 @@ export interface CursorTrackPoint {
 	virtualSec?: number | null;
 	cx: number;
 	cy: number;
+	/** Present only while the recorded pointer was hidden. Absent means visible. */
+	visible?: false;
 	/** Small stable index per distinct pointer shape within THIS track. Absent when
 	 *  the recording carries no shape information. */
 	shape?: number;
@@ -68,7 +72,7 @@ export interface CursorTrack {
 	truncated: boolean;
 	/** Present ONLY when the ceiling did not hold. `maxPoints` budgets the rate and
 	 *  the gap floor; the MANDATORY points are exempt and stack on top — the first
-	 *  and last sample, a pointer-shape change, a non-move event, the ends of a run
+	 *  and last sample, a pointer-shape or visibility change, a non-move event, the ends of a run
 	 *  longer than the max gap — so a capture rich in them lands above the ceiling.
 	 *  Absent means the budget held. It is a separate field from `truncated` on
 	 *  purpose: that one says "you are seeing less than you asked for", this one
@@ -240,8 +244,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {
 	// saving — 356 grid points → 170. A parked pointer collapses for the same reason: the
 	// chord through a stationary run is a point, so every in-between lies on it.
 	//
-	// Three things outrank the tolerance and are never dropped, because no interpolation
-	// puts them back: a pointer-shape change, a non-move event, and the ends of a run
+	// Four things outrank the tolerance and are never dropped, because no interpolation
+	// puts them back: a pointer-shape change, a visibility change, a non-move event, and the ends of a run
 	// longer than `maxGapMs` (a parked cursor must read as "still here", never as missing
 	// data). `minIntervalMs` then caps the rate so a thrashing pointer cannot spend the
 	// whole budget in one second.
@@ -252,16 +256,19 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {
 		mandatory.add(ordered.length - 1);
 	}
 	let lastShape: string | null | undefined = ordered[0]?.assetId;
+	let lastVisible = ordered[0]?.visible !== false;
 	let lastMandatoryMs = ordered[0]?.timeMs ?? 0;
 	for (let i = 0; i < ordered.length; i += 1) {
 		const s = ordered[i];
 		const shapeChanged = s.assetId !== lastShape && shapeIndex.size > 1;
+		const visibilityChanged = (s.visible !== false) !== lastVisible;
 		const notAMove = typeof s.interactionType === "string" && s.interactionType !== "move";
 		const stale = s.timeMs - lastMandatoryMs >= maxGapMs;
-		if (shapeChanged || notAMove || stale) {
+		if (shapeChanged || visibilityChanged || notAMove || stale) {
 			mandatory.add(i);
 			lastMandatoryMs = s.timeMs;
 			lastShape = s.assetId;
+			lastVisible = s.visible !== false;
 		}
 	}
 
@@ -309,6 +316,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {
 			cx: round3(s.cx),
 			cy: round3(s.cy),
 		};
+		if (s.visible === false) point.visible = false;
 		if (shifted) point.virtualSec = position ? round2(position.virtualTimeSec) : null;
 		const shape = typeof s.assetId === "string" ? shapeIndex.get(s.assetId) : undefined;
 		if (shape !== undefined && shapeIndex.size > 1) point.shape = shape;
@@ -330,7 +338,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {
 	const overBudget =
 		points.length > maxPoints
 			? `${points.length} points for a ceiling of ${maxPoints}: the mandatory points are ` +
-				`never dropped — the first and last sample, pointer-shape changes, non-move ` +
+				`never dropped — the first and last sample, pointer-shape and visibility changes, non-move ` +
 				`events and the ends of a parked run — and this recording has enough of them ` +
 				`to land above the budget.`
 			: undefined;
@@ -352,7 +360,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack {
 			"the points. A null virtualSec means no clip carries that moment; trimmed:true means a " +
 			"trim cuts it out of " +
 			"playback, so a zoom there would never be seen. `shape` is an index into the pointer " +
-			"bitmaps this recording used: equal values are the same pointer, a change is a change.",
+			"bitmaps this recording used: equal values are the same pointer, a change is a change. " +
+			"visible:false marks a hidden pointer; absent means visible, including older recordings.",
 		points,
 	};
 }

technical-documentation/architecture/ai-agent.md

@@ -47,7 +47,7 @@ The model never free-writes the project document. It can only call the fixed set
 |---|---|---|
 | `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, and `cutByJunction` when a clip junction does (the compositor renders each clip on its own, so a move running past its clip's edge is lost too), so the export jumps there (#1028). A side cut by both is blamed on the cut nearest the pill, where the jump is. Chained zooms (at most 1.5 s apart on screen, no trim or clip junction between them) report the pan instead: `panToZoomId` / `panUntilSec` on the first, `panFromZoomId` on the second, in place of its zoom-out and zoom-in (`chainedPans`). | Nothing. |
 | `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. |
-| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. |
+| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. `visible:false` marks a hidden pointer; absent means visible, including older sidecars. Visibility changes survive downsampling even when the pointer stays still. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. |
 | `addTrim` | Adds one source-time cut inside a clip. Like the zoom writes and `setTrim`, it reports `cutTransitions` (zoom, side, seconds lost, `by` trim or junction) when the edit newly cuts a zoom's move in or out; a warning, never a refusal. | `timeline.trimRanges`. |
 | `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. |
 | `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. |