From 9ed89e5b8a6977a13cb095a28a2b5a3dc8b7db73 Mon Sep 17 00:00:00 2001 From: Satwik Sharma <167347544+satwiksharma01@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:03:10 +0530 Subject: [PATCH 1/2] fix(agent): preserve recorded cursor visibility --- electron/ai-edition/agent-tools.test.ts | 23 ++++++++++++ .../ai-edition/timeline/cursor-track.test.ts | 37 +++++++++++++++++++ src/lib/ai-edition/timeline/cursor-track.ts | 23 ++++++++---- .../architecture/ai-agent.md | 2 +- 4 files changed, 77 insertions(+), 8 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 96bdc852c..10a3af4c4 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1686,6 +1686,29 @@ describe("getCursorTrack", () => { expect(result.resultJson).not.toContain('virtualSec":'); }); + it("reports a hidden click and the return to visibility through the tool payload", () => { + const samples = [ + { timeMs: 0, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + { timeMs: 50, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }, + { timeMs: 100, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + ]; + const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", { + cursorTelemetry: { load: { status: "ok", assetId: "asset_1", samples } }, + }); + const payload = JSON.parse(result.resultJson); + + expect(result.ok).toBe(true); + expect(payload.points).toContainEqual({ + atSec: 0.05, + cx: 0.5, + cy: 0.5, + kind: "click", + visible: false, + }); + expect(payload.points).toContainEqual({ atSec: 0.1, cx: 0.5, cy: 0.5 }); + }); + it("says 'no-sidecar' when the asset was checked and has none", () => { const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", { cursorTelemetry: { load: { status: "no-sidecar", assetId: "asset_1" } }, diff --git a/src/lib/ai-edition/timeline/cursor-track.test.ts b/src/lib/ai-edition/timeline/cursor-track.test.ts index b773e099b..941d10d65 100644 --- a/src/lib/ai-edition/timeline/cursor-track.test.ts +++ b/src/lib/ai-edition/timeline/cursor-track.test.ts @@ -76,6 +76,43 @@ describe("buildCursorTrack", () => { expect(click?.atSec).toBeCloseTo(13 * 0.05, 2); }); + it("keeps both visibility changes even when a stationary cursor is sampled below their rate", () => { + const samples = Array.from({ length: 8 }, (_, i) => ({ + timeMs: i * 50, + cx: 0.5, + cy: 0.5, + assetId: "arrow", + interactionType: "move", + visible: i === 3 ? false : undefined, + })); + const track = build(samples, 0.1); + + expect(track.points.find((point) => point.atSec === 0.15)).toMatchObject({ visible: false }); + expect(track.points.find((point) => point.atSec === 0.2)).toBeDefined(); + expect(track.points.find((point) => point.atSec === 0.2)).not.toHaveProperty("visible"); + }); + + it("marks a recorded click as hidden instead of presenting it as visible to the agent", () => { + const samples = [ + { timeMs: 0, cx: 0.5, cy: 0.5, interactionType: "move", visible: true }, + { timeMs: 500, cx: 0.5, cy: 0.5, interactionType: "click", visible: false }, + { timeMs: 1000, cx: 0.5, cy: 0.5, interactionType: "move", visible: true }, + ]; + expect(build(samples).points.find((point) => point.kind === "click")).toMatchObject({ + atSec: 0.5, + visible: false, + }); + }); + + it("treats legacy samples without visibility as visible, omitting the default field", () => { + const legacy = sweep(40); + const explicit = legacy.map((sample) => ({ ...sample, visible: true })); + const legacyTrack = build(legacy); + + expect(build(explicit).points).toEqual(legacyTrack.points); + expect(legacyTrack.points.every((point) => !("visible" in point))).toBe(true); + }); + it("drops to a coarser rate rather than blowing the ceiling, and says so", () => { // 40 minutes at 20 Hz: 5 Hz would be 12 000 points. const track = buildCursorTrack({ diff --git a/src/lib/ai-edition/timeline/cursor-track.ts b/src/lib/ai-edition/timeline/cursor-track.ts index b82db3648..6a700f383 100644 --- a/src/lib/ai-edition/timeline/cursor-track.ts +++ b/src/lib/ai-edition/timeline/cursor-track.ts @@ -1,5 +1,5 @@ // The recorded cursor track the agent reads: where the pointer was, when, and -// what shape it had. Pure — no fs, no IPC. The caller supplies the samples. +// what shape it had, and when it was visible. Pure — no fs, no IPC. The caller supplies the samples. // // ponytail: this is an OBSERVATION, not an interpretation, and the distinction // is the whole point of the module. Its predecessor handed the model a list of @@ -28,6 +28,8 @@ export interface CursorTrackSample { timeMs: number; cx: number; cy: number; + /** Absent in older sidecars, where the pointer is treated as visible. */ + visible?: boolean; /** The cursor BITMAP's id, not a media asset: the sidecar stores one entry per * distinct pointer image (arrow, hand, text caret, resize…). A change means the * pointer shape changed, which is why these points are never dropped. */ @@ -44,6 +46,8 @@ export interface CursorTrackPoint { virtualSec?: number | null; cx: number; cy: number; + /** Present only while the recorded pointer was hidden. Absent means visible. */ + visible?: false; /** Small stable index per distinct pointer shape within THIS track. Absent when * the recording carries no shape information. */ shape?: number; @@ -68,7 +72,7 @@ export interface CursorTrack { truncated: boolean; /** Present ONLY when the ceiling did not hold. `maxPoints` budgets the rate and * the gap floor; the MANDATORY points are exempt and stack on top — the first - * and last sample, a pointer-shape change, a non-move event, the ends of a run + * and last sample, a pointer-shape or visibility change, a non-move event, the ends of a run * longer than the max gap — so a capture rich in them lands above the ceiling. * Absent means the budget held. It is a separate field from `truncated` on * purpose: that one says "you are seeing less than you asked for", this one @@ -240,8 +244,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack { // saving — 356 grid points → 170. A parked pointer collapses for the same reason: the // chord through a stationary run is a point, so every in-between lies on it. // - // Three things outrank the tolerance and are never dropped, because no interpolation - // puts them back: a pointer-shape change, a non-move event, and the ends of a run + // Four things outrank the tolerance and are never dropped, because no interpolation + // puts them back: a pointer-shape change, a visibility change, a non-move event, and the ends of a run // longer than `maxGapMs` (a parked cursor must read as "still here", never as missing // data). `minIntervalMs` then caps the rate so a thrashing pointer cannot spend the // whole budget in one second. @@ -252,16 +256,19 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack { mandatory.add(ordered.length - 1); } let lastShape: string | null | undefined = ordered[0]?.assetId; + let lastVisible = ordered[0]?.visible !== false; let lastMandatoryMs = ordered[0]?.timeMs ?? 0; for (let i = 0; i < ordered.length; i += 1) { const s = ordered[i]; const shapeChanged = s.assetId !== lastShape && shapeIndex.size > 1; + const visibilityChanged = (s.visible !== false) !== lastVisible; const notAMove = typeof s.interactionType === "string" && s.interactionType !== "move"; const stale = s.timeMs - lastMandatoryMs >= maxGapMs; - if (shapeChanged || notAMove || stale) { + if (shapeChanged || visibilityChanged || notAMove || stale) { mandatory.add(i); lastMandatoryMs = s.timeMs; lastShape = s.assetId; + lastVisible = s.visible !== false; } } @@ -309,6 +316,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack { cx: round3(s.cx), cy: round3(s.cy), }; + if (s.visible === false) point.visible = false; if (shifted) point.virtualSec = position ? round2(position.virtualTimeSec) : null; const shape = typeof s.assetId === "string" ? shapeIndex.get(s.assetId) : undefined; if (shape !== undefined && shapeIndex.size > 1) point.shape = shape; @@ -330,7 +338,7 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack { const overBudget = points.length > maxPoints ? `${points.length} points for a ceiling of ${maxPoints}: the mandatory points are ` + - `never dropped — the first and last sample, pointer-shape changes, non-move ` + + `never dropped — the first and last sample, pointer-shape and visibility changes, non-move ` + `events and the ends of a parked run — and this recording has enough of them ` + `to land above the budget.` : undefined; @@ -352,7 +360,8 @@ export function buildCursorTrack(options: CursorTrackOptions): CursorTrack { "the points. A null virtualSec means no clip carries that moment; trimmed:true means a " + "trim cuts it out of " + "playback, so a zoom there would never be seen. `shape` is an index into the pointer " + - "bitmaps this recording used: equal values are the same pointer, a change is a change.", + "bitmaps this recording used: equal values are the same pointer, a change is a change. " + + "visible:false marks a hidden pointer; absent means visible, including older recordings.", points, }; } diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md index 180402381..5136f0ae2 100644 --- a/technical-documentation/architecture/ai-agent.md +++ b/technical-documentation/architecture/ai-agent.md @@ -47,7 +47,7 @@ The model never free-writes the project document. It can only call the fixed set |---|---|---| | `getCurrentDocument` | Reads a compact project, asset, clip, trim, and modifier snapshot with explicit time bases. Each asset reports `hasCameraTrack` / `cameraVisible` / `hasCursorTelemetry` beside `hasTranscript` (`hasCursorTelemetry` is three-valued: `true`, `false` when the asset was checked and has none, `null` when it was not checked — never `false` for something we failed to look at), the document reports `hasAnyCamera` and `autoFocusAll`, and each zoom reports the `renderedScale` the viewer will see plus `customScale` / `depthIsOverridden` when a custom scale makes its `depth` inert. Each zoom also reports where its camera moves sit on the ruler, outside the pill: `zoomInFromSec`, `zoomOutUntilSec` (speed-aware, from `zoomTransitions` in `src/lib/ai-edition/timeline/zoom-transitions.ts`), `transitionSec`, `cutByTrim` (`in` / `out` / `both`) when a trim cuts a move, and `cutByJunction` when a clip junction does (the compositor renders each clip on its own, so a move running past its clip's edge is lost too), so the export jumps there (#1028). A side cut by both is blamed on the cut nearest the pill, where the jump is. Chained zooms (at most 1.5 s apart on screen, no trim or clip junction between them) report the pan instead: `panToZoomId` / `panUntilSec` on the first, `panFromZoomId` on the second, in place of its zoom-out and zoom-in (`chainedPans`). | Nothing. | | `getTranscript` | Reads the transcript segments for an asset, or the primary asset, in full. On the production path a segment is one word, so a half-hour recording is a few thousand of them — there is no cap, and no per-model context budget to derive one from. | Nothing. | -| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | +| `getCursorTrack` | Reads the recorded pointer telemetry for an asset as REAL SAMPLES, reduced per axis against time (Douglas–Peucker) rather than summarised: every point returned is one that was recorded, carrying its source `atSec`, `cx`/`cy`, the `virtualSec` that `addZoom` takes (left off the points, with `virtualEqualsSource` on the envelope, when the two coincide everywhere), a `shape` index per distinct pointer bitmap, and `kind` / `trimmed` where they apply. `visible:false` marks a hidden pointer; absent means visible, including older sidecars. Visibility changes survive downsampling even when the pointer stays still. Answers `available:false` with `reason:"no-sidecar"` (checked, this asset has none) or `reason:"unavailable"` (could not be read from here), and the two are never conflated. | Nothing. | | `addTrim` | Adds one source-time cut inside a clip. Like the zoom writes and `setTrim`, it reports `cutTransitions` (zoom, side, seconds lost, `by` trim or junction) when the edit newly cuts a zoom's move in or out; a warning, never a refusal. | `timeline.trimRanges`. | | `addTrims` | Adds many cuts in one call, replaying `addTrim` per entry so the rules cannot drift apart. Each range stands alone: one that cannot be placed is refused by itself and named with its index and reason while the rest are applied, and the result leads with `requested` / `appliedCount` / `refusedCount`. Only a batch where nothing landed is an error. | `timeline.trimRanges`. | | `setTrim` | Moves or resizes an existing source-time trim. | The matching `timeline.trimRanges` entry. | From a3918d88e4cd4748c06fb23810a951f7d3115900 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Sat, 10 Oct 2026 10:40:28 +0200 Subject: [PATCH 2/2] docs(agent): name visible:false in the getCursorTrack tool description --- electron/ai-edition/deep-agent/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 81b084824..88a561679 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -154,7 +154,7 @@ export const TOOL_DESCRIPTIONS: Record = { getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: - "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", + "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`; points where the pointer was hidden carry `visible:false`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", getTranscriptWords: 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', setWordText: